def get_filename(maintitle, season, title):
try:
match = re.compile(
r'(?P<title>.*?)\s?((?P<season>\d+)기)?\s?((?P<epi_no>\d+)화)'
).search(title)
if match:
epi_no = int(match.group('epi_no'))
if epi_no < 10:
epi_no = '0%s' % epi_no
else:
epi_no = '%s' % epi_no
if int(season) < 10:
season = '0%s' % season
else:
season = '%s' % season
#title_part = match.group('title').strip()
#ret = '%s.S%sE%s%s.720p-SA.mp4' % (maintitle, season, epi_no, date_str)
ret = '%s S%sE%s.mp4' % (maintitle, season, epi_no)
else:
logger.debug('NOT MATCH')
ret = '%s.720p-SA.mp4' % title
return Util.change_text_for_use_filename(ret)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def queue_append(queue_list):
try:
logger.debug(queue_list)
new_queue_list = []
for q in queue_list:
src, tar = q.split('|')
tmps = tar.split('/')
if len(tmps) > 1:
for i in range(1, len(tmps)):
tmps[i] = Util.change_text_for_use_filename(
tmps[i]).replace(' ', ' ').replace(
' ', ' ').rstrip('.').strip()
new_queue_list.append('%s|%s/%s' %
(src, tmps[0], '/'.join(tmps[1:])))
else:
new_queue_list.append(q)
logger.debug(new_queue_list)
tmp = ModelSetting.get('gclone_queue_list')
tmp += '\n' + '\n'.join(new_queue_list)
ModelSetting.set('gclone_queue_list', tmp)
socketio_callback('refresh_queue',
ModelSetting.get('gclone_queue_list'))
return LogicGclone.start()
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def apply_new_title(new_title):
try:
ret = {}
if LogicLinkkf.current_data is not None:
program = db.session.query(ModelLinkkfProgram) \
.filter_by(programcode=LogicLinkkf.current_data['code']) \
.first()
new_title = Util.change_text_for_use_filename(new_title)
LogicLinkkf.current_data['save_folder'] = new_title
program.save_folder = new_title
db.session.commit()
for entity in LogicLinkkf.current_data['episode']:
entity['save_folder'] = new_title
entity['filename'] = LogicLinkkf.get_filename(
LogicLinkkf.current_data['save_folder'],
LogicLinkkf.current_data['season'], entity['title'])
# tmp = data['filename'].split('.')
# tmp[0] = new_title
# data['filename'] = '.'.join(tmp)
return LogicLinkkf.current_data
else:
ret['ret'] = False
ret['log'] = 'No current data!!'
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
ret['ret'] = False
ret['log'] = str(e)
return ret
def get_title_info(code):
try:
if LogicAni24.current_data is not None and LogicAni24.current_data['code'] == code and LogicAni24.current_data['ret']:
return LogicAni24.current_data
url = '%s/ani_list/%s.html' % (ModelSetting.get('ani24_url'), code)
data = LogicAni24.get_html(url)
tree = html.fromstring(data)
data = {}
data['code'] = code
data['ret'] = False
tmp = tree.xpath('//h1[@class="ani_info_title_font_box"]')[0].text_content().strip().encode('utf8')
match = re.compile(r'(?P<season>\d+)기').search(tmp)
if match:
data['season'] = match.group('season')
else:
data['season'] = '1'
data['title'] = tmp.replace(data['season']+u'기', '').strip()
data['title'] = Util.change_text_for_use_filename(data['title']).replace('OVA', '').strip()
try:
data['poster_url'] = 'https:' + tree.xpath('//div[@class="ani_info_left_box"]/img')[0].attrib['src']
data['detail'] = []
tmp = tree.xpath('//div[@class="ani_info_right_box"]/div')
for t in tmp:
detail = t.xpath('.//span')
data['detail'].append({detail[0].text_content().strip():detail[-1].text_content().strip()})
except:
data['detail'] = [{'정보없음':''}]
data['poster_url'] = None
tmp = tree.xpath('//span[@class="episode_count"]')[0].text_content().strip()
match = re.compile(r'\d+').search(tmp)
if match:
data['episode_count'] = match.group(0)
else:
data['episode_count'] = '0'
data['episode'] = []
tags = tree.xpath('//div[@class="ani_video_list"]/a')
re1 = re.compile(r'ani_view\/(?P<code>\d+)\.html')
for t in tags:
entity = {}
entity['code'] = re1.search(t.attrib['href']).group('code')
data['episode'].append(entity)
tmp = t.xpath('.//img')[0]
entity['image'] = 'https:' + tmp.attrib['src']
tmp = t.xpath('.//div[2]/div')
entity['title'] = tmp[0].text_content().strip().encode('utf8')
entity['date'] = tmp[1].text_content().strip()
entity['filename'] = LogicAni24.get_filename(data['title'], entity['title'],entity['date'])
data['ret'] = True
LogicAni24.current_data = data
return data
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
data['log'] = str(e)
return data
def apply_new_title(new_title):
try:
ret = {}
if LogicAni24.current_data is not None:
new_title = Util.change_text_for_use_filename(new_title)
LogicAni24.current_data['title'] = new_title
for data in LogicAni24.current_data['episode']:
tmp = data['filename'].split('.')
tmp[0] = new_title
data['filename'] = '.'.join(tmp)
return LogicAni24.current_data
else:
ret['ret'] = False
ret['log'] = 'No current data!!'
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
ret['ret'] = False
ret['log'] = str(e)
return ret
def get_filename(maintitle, title, date):
try:
match = re.compile(
r'(?P<title>.*?)\s?((?P<season>\d+)기)?\s?((?P<epi_no>\d[\d]*)(?=[^\d]).*화)'
).search(title)
if match:
if match.group('season') is not None:
season = int(match.group('season'))
if season < 10:
season = '0%s' % season
else:
season = '%s' % season
else:
season = '01'
epi_no = int(match.group('epi_no'))
if epi_no < 10:
epi_no = '0%s' % epi_no
else:
epi_no = '%s' % epi_no
#date 옵션
if ModelSetting.get('include_date') == 'True':
if ModelSetting.get('date_option') == '1':
date_str = '.%s' % date
elif ModelSetting.get('date_option') == '0':
date_str = '.' + date[2:4] + date[5:7] + date[8:10]
else:
date_str = ''
#title_part = match.group('title').strip()
ret = '%s.S%sE%s%s.720p-SA.mp4' % (maintitle, season, epi_no,
date_str)
else:
logger.debug('NOT MATCH')
ret = '%s.720p-SA.mp4' % title
return Util.change_text_for_use_filename(ret)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
def make_episode_info(self):
try:
data = requests.get(
'https://www.aniplustv.com/aniplus2020Api/base64encode.asp?codeString={userid}'
.format(userid=ModelSetting.get('aniplus_id')),
headers=headers).text
userid2 = data.replace('{"codeString":"', '').replace('"}', '')
data = requests.get(
'https://www.aniplustv.com/aniplus2020Api/base64encode.asp?codeString={time}|was|{subPartSerial2}/{userid}@{userid2}'
.format(time=(datetime.now() +
timedelta(hours=1)).strftime('%Y%m%d%H%M%S'),
subPartSerial2=self.info['subPartSerial2'],
userid=ModelSetting.get('aniplus_id'),
userid2=userid2),
headers=headers).text
crypParam = data.replace('{"codeString":"', '').replace('"}', '')
data = requests.post('https://api.aniplustv.com:3100/vodUrl',
headers=headers,
json={
"params": {
"userid":
ModelSetting.get('aniplus_id'),
"subPartSerial":
self.info['subPartSerial2'],
"crypParam":
crypParam,
'authId':
base64.b64encode(
ModelSetting.get('aniplus_id')),
'token':
get_token()
}
}).json()[0]
if 'use1080' in data and data['use1080'] != '':
self.quality = '1080p'
self.url = data['use1080']
elif 'fileName1080p' in data and data['fileName1080p'] != '':
self.quality = '1080p'
self.url = data['fileName1080p']
elif 'fileName720p' in data and data['fileName720p'] != '':
self.quality = '720p'
self.url = data['fileName720p']
elif 'fileName480p' in data and data['fileName480p'] != '':
self.quality = '480p'
self.url = data['fileName480p']
else:
return False
match = re.compile(r'(?P<title>.*?)\s*((?P<season>\d+)%s)' %
(u'기')).search(self.info['title'])
if match:
content_title = match.group('title').strip()
if 'season' in match.groupdict() and match.group(
'season') is not None:
self.season = int(match.group('season'))
else:
content_title = self.info['title']
self.season = 1
if content_title.find(u'극장판') != -1:
ret = '%s.%s-SAP.mp4' % (content_title, self.quality)
else:
ret = '%s.S%sE%s.%s-SAP.mp4' % (content_title, str(
self.season).zfill(2), str(
self.info['part']).zfill(2), self.quality)
self.filename = Util.change_text_for_use_filename(ret)
self.savepath = P.ModelSetting.get('aniplus_download_path')
if P.ModelSetting.get_bool('aniplus_auto_make_folder'):
folder_name = Util.change_text_for_use_filename(
content_title.strip())
self.savepath = os.path.join(self.savepath, folder_name)
self.filepath = os.path.join(self.savepath, self.filename)
if not os.path.exists(self.savepath):
os.makedirs(self.savepath)
return True
except Exception as e:
P.logger.error('Exception:%s', e)
P.logger.error(traceback.format_exc())
def get_title_info(code):
try:
if LogicLinkkf.current_data is not None and LogicLinkkf.current_data[
'code'] == code and LogicLinkkf.current_data['ret']:
return LogicLinkkf.current_data
url = '%s/%s' % (ModelSetting.get('linkkf_url'), code)
data = LogicLinkkf.get_html(url)
tree = html.fromstring(data)
data = {}
data['code'] = code
data['ret'] = False
tmp = tree.xpath('/html/body/div[2]/div/div/article/center/strong'
)[0].text_content().strip().encode('utf8')
match = re.compile(r'(?P<season>\d+)기').search(tmp)
if match:
data['season'] = match.group('season')
else:
data['season'] = '1'
data['title'] = tmp.replace(data['season'] + u'기', '').strip()
data['title'] = Util.change_text_for_use_filename(
data['title']).replace('OVA', '').strip()
try:
data['poster_url'] = tree.xpath(
'//*[@id="body"]/div/div/div[1]/center/img'
)[0].attrib['data-src']
data['detail'] = [{
'info':
tree.xpath('/html/body/div[2]/div/div/div[1]')
[0].text_content().strip().encode('utf8')
}]
except:
data['detail'] = [{'정보없음': ''}]
data['poster_url'] = None
tmp = tree.xpath('//button')
if tmp is not None:
data['episode_count'] = len(tmp)
else:
data['episode_count'] = '0'
data['episode'] = []
tags = tree.xpath('//button/a')
data['save_folder'] = data['title']
program = db.session.query(ModelLinkkfProgram) \
.filter_by(programcode=code) \
.first()
if (program is None):
program = ModelLinkkfProgram(data)
db.session.add(program)
db.session.commit()
else:
data['save_folder'] = program.save_folder
data['season'] = program.season
for t in tags:
entity = {}
entity['program_code'] = data['code']
entity['program_title'] = data['title']
entity['save_folder'] = Util.change_text_for_use_filename(
data['save_folder'])
entity['code'] = urlparse.urlsplit(t.attrib['href']).path
data['episode'].append(entity)
entity['image'] = data['poster_url']
entity['title'] = t.text_content().strip().encode('utf8')
entity['filename'] = LogicLinkkf.get_filename(
data['save_folder'], data['season'], entity['title'])
data['ret'] = True
LogicLinkkf.current_data = data
return data
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
data['log'] = str(e)
return data
def make_episode_info(self):
try:
url = 'https://www.jetcloud-list.cc/kr/episode/' + self.info['va']
text = requests.get(url, headers=headers).text
match = re.compile('src\=\"(?P<video_url>http.*?\.m3u8)').search(text)
if match:
tmp = match.group('video_url')
# 2020-11-06 master.m3u8 cloudflare가 막음. 화질별은 아직 안막음.
#m3u8 = requests.get(tmp, headers=LogicAni365.current_headers).text
#for t in m3u8.split('\n'):
# if t.find('m3u8') != -1:
# self.url = tmp.replace('master.m3u8', t.strip())
# self.quality = t.split('.m3u8')[0]
m3u8_text = requests.get(tmp, headers=headers).text.strip()
self.url = m3u8_text.split('\n')[-1].strip()
logger.debug(self.url)
self.quality = self.url.split('/')[-1].split('.')[0]
"""
logger.debug(tmp)
master = tmp.replace('https://', '').split('/')
logger.debug(master)
master[1] += 's'
url_1080 = copy.deepcopy(master)
url_1080.insert(3, '1080')
url_1080[-1] = url_1080[-1].replace('master', '1080')
tmp = 'https://' + '/'.join(url_1080)
res = requests.get(tmp, headers=LogicAni365.current_headers)
if res.status_code == 200:
self.url = tmp
self.quality = '1080'
else:
url_720 = copy.deepcopy(master)
url_720.insert(3, '720')
url_720[-1] = url_1080[-1].replace('master', '720')
self.url = 'https://' + '/'.join(url_720)
self.quality = '720'
"""
#https://www.jetcloud-list.cc/getfiles/4yekl4kluyjldcefts7wuNtfQ7tRhoqyywN08Qb1bbg5ja32gv/1080/9cfea65b412beb6d02cda008326ec9d2/1080.m3u8
#https://www.jetcloud-list.cc/getfiles/uwcngQuksgs5fka9qe2eg7tPdkhNejchht6xija5dcqtafthjj/1080/9cfea65b412beb6d02cda008326ec9d2/1080.m3u8
match = re.compile(r'src\=\"(?P<vtt_url>http.*?kr.vtt)').search(text)
if match:
self.vtt = u'%s' % match.group('vtt_url')
match = re.compile(r'(?P<title>.*?)\s*((?P<season>\d+)%s)?\s*((?P<epi_no>\d+)%s)' % (u'기', u'화')).search(self.info['title'])
if match:
self.content_title = match.group('title').strip()
if 'season' in match.groupdict() and match.group('season') is not None:
self.season = int(match.group('season'))
epi_no = int(match.group('epi_no'))
ret = '%s.S%sE%s.%s-SA.mp4' % (self.content_title, '0%s' % self.season if self.season < 10 else self.season, '0%s' % epi_no if epi_no < 10 else epi_no, self.quality)
else:
self.content_title = self.info['title']
P.logger.debug('NOT MATCH')
ret = '%s.720p-SA.mp4' % self.info['title']
self.filename = Util.change_text_for_use_filename(ret)
self.savepath = P.ModelSetting.get('ani365_download_path')
if P.ModelSetting.get_bool('ani365_auto_make_folder'):
if self.info['day'].find(u'완결') != -1:
folder_name = '%s %s' % (P.ModelSetting.get('ani365_finished_insert'), self.content_title)
else:
folder_name = self.content_title
folder_name = Util.change_text_for_use_filename ( folder_name.strip() )
self.savepath = os.path.join(self.savepath, folder_name)
if P.ModelSetting.get_bool('ani365_auto_make_season_folder'):
self.savepath = os.path.join(self.savepath, 'Season %s' % int(self.season))
self.filepath = os.path.join(self.savepath, self.filename)
if not os.path.exists(self.savepath):
os.makedirs(self.savepath)
from framework.common.util import write_file, convert_vtt_to_srt
srt_filepath = os.path.join(self.savepath, self.filename.replace('.mp4', '.ko.srt'))
if not os.path.exists(srt_filepath):
vtt_data = requests.get(self.vtt, headers=LogicAni365.current_headers).text
srt_data = convert_vtt_to_srt(vtt_data)
write_file(srt_data, srt_filepath)
self.headers = LogicAni365.current_headers
except Exception as e:
P.logger.error('Exception:%s', e)
P.logger.error(traceback.format_exc())
def download(entity):
try:
entity['download_count'] += 1
entity['status'] = 1
entity['str_status'] = '대기'
LogicNormal.entity_update(entity)
url = 'http://webtoon.daum.net/data/pc/webtoon/viewer_images/%s' % (
entity['episode_id'])
data = requests.get(url).json()
entity['str_status'] = '분석'
LogicNormal.entity_update(entity)
dirname = ModelSetting.get('download_path')
if ModelSetting.get_bool('use_title_folder'):
dirname = os.path.join(
dirname,
Util.change_text_for_use_filename(entity['toon_title']))
#if not os.path.exists(dirname):
# os.makedirs(dirname)
tmp = u'%s %s %s' % (entity['episode_idx'].zfill(3),
entity['toon_title'], entity['episode_title'])
dirname = os.path.join(dirname,
Util.change_text_for_use_filename(tmp))
if not os.path.exists(dirname):
os.makedirs(dirname)
entity['filename'] = '%s.zip' % dirname
if os.path.exists(entity['filename']):
entity['status'] = 12
entity['str_status'] = '파일 있음'
LogicNormal.entity_update(entity)
else:
entity['str_status'] = '다운로드중'
LogicNormal.entity_update(entity)
count = len(data['data'])
for idx, tmp in enumerate(data['data']):
filename = os.path.join(dirname,
str(idx + 1).zfill(2) + '.jpg')
image_data = requests.get(tmp['url'],
headers=headers,
stream=True)
with open(filename, 'wb') as handler:
handler.write(image_data.content)
entity['str_status'] = '다운로드중 %s / %s' % (idx + 1, count)
entity['percent'] = int(100.0 * (idx + 1) / count)
LogicNormal.entity_update(entity)
Util.makezip(dirname)
entity['status'] = 11
entity['str_status'] = '완료'
LogicNormal.entity_update(entity)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
entity['status'] = 2
entity['str_status'] = '실패'
if entity['download_count'] >= 5:
entity['status'] = 13
entity['str_status'] = '재시도초과'
LogicNormal.entity_update(entity)
ModelItem.save_as_dict(entity)
def download(self):
completed_flag = False
try:
self.set_status(u'분석중')
url = '{}/{}/{}'.format(
ModelSetting.get('{}_url'.format(self.module_logic.name)),
define[module_name]['url_prefix'], self.info['code'])
data = requests.get(url).text
tmp = ''.join(
re.compile(r'html_data\+\=\'(.*?)\'\;').findall(data))
html = ''.join(
[chr(int(x, 16)) for x in tmp.rstrip('.').split('.')])
image_list = re.compile(
r'img\ssrc="/img/loading-image.gif"\sdata\-\w{11}="(.*?)"'
).findall(html)
self.total_image_count = len(image_list)
self.refresh_status()
download_folder = ModelSetting.get(
'{}_download_folder'.format(module_name))
if ModelSetting.get_bool(
'{}_make_series_folder'.format(module_name)):
download_folder = os.path.join(
download_folder,
Util.change_text_for_use_filename(
self.info['series_title']))
self.savepath = os.path.join(
download_folder,
Util.change_text_for_use_filename(self.info['title']))
if not os.path.exists(self.savepath):
os.makedirs(self.savepath)
zip_extension = ModelSetting.get(
'{}_zip_extension'.format(module_name))
if ModelSetting.get(
'{}_use_zip'.format(module_name)) and os.path.exists(
self.savepath + '.' + zip_extension):
self.percent = 100
self.set_status(u'파일 있음')
self.savepath = self.savepath + '.' + zip_extension
completed_flag = True
return
self.set_status(u'다운로드중')
for idx, tmp in enumerate(image_list):
try:
ext = tmp.split('/')[-1].split('.')[-1]
if len(ext) > 4:
ext = 'jpg'
except:
pass
url = tmp
if not url.startswith('http'):
url = '%s%s' % (ModelSetting.get('{}_url'.format(
self.module_logic.name)), tmp)
filepath = os.path.join(self.savepath,
str(idx + 1).zfill(3) + '.' + ext)
ret = self.image_download(url, filepath)
# 실패처리
if ret != 200:
ret = self.image_download(url, filepath)
if ret != 200:
self.set_status(u'실패')
shutil.rmtree(self.savepath)
return
self.percent = (int)(
(idx + 1) * 100 / self.total_image_count)
self.refresh_status()
if ModelSetting.get('{}_use_zip'.format(module_name)):
self.set_status(u'압축파일 생성중')
Util.makezip(self.savepath, zip_extension=zip_extension)
self.savepath = self.savepath + '.' + zip_extension
self.set_status(u'다운로드 완료')
completed_flag = True
except Exception as e:
P.logger.error('Exception:%s', e)
P.logger.error(traceback.format_exc())
self.set_status(u'실패')
finally:
if completed_flag:
item = ModelItem.get_by_code(self.info['code'])
item.savepath = self.savepath
item.status = 'completed'
item.completed_time = datetime.now()
item.save()
if ModelSetting.get_bool(
'{}_queue_auto_clear'.format(module_name)):
self.module_logic.queue.command('delete_completed', -1)
self.module_logic.socketio_callback('list_refresh', '')
def get_filename(data, quality):
try:
if 'movieid' in data:
title = Util.change_text_for_use_filename(data['title'])
releasedate = data['releasedate'][:4]
ret = "%s.%s.%s-SW.mp4" % (title, releasedate, quality)
return ret
else:
title = Util.change_text_for_use_filename(data['programtitle'])
tmp = data["episodenumber"]
episodeno = ''
if len(tmp):
if tmp == u'특집':
episodeno = u'.특집'
else:
tmps = tmp.split('-')
if len(tmps[0]) < 6:
try:
is_number = int(tmps[0])
episodeno = '.E0%s' % tmps[0] if len(tmps[0]) == 1 else '.E%s' % tmps[0]
except:
episodeno = u'.%s' % tmps[0]
airdate = data["releasedate"].replace('-', '')[2:]
release = "SW"
if data['type'] == 'onair':
release = 'SWQ'
ret = "%s%s.%s.%s-%s.mp4" % (title, episodeno, airdate, quality, release)
return ret
if episode_data['result']['contentType'] == 'movie':
title = episode_data["result"]["programTitle"]
title = title.replace("<", "").replace(">", "").replace("\\", "").replace("/", "").replace(":", "").replace("*", "").replace("\"", "").replace("|", "").replace("?", "").replace(" ", " ").strip()
currentQuality = episode_data["result"]["qualityList"]['qualityCurrent']
qualityRes = PooqAPI.get_quality_to_res(currentQuality)
airDate = episode_data["result"]["airDate"]
ret = "%s.%s.%s-SP.mp4" % (title, airDate[:4], qualityRes)
return ret
else:
title = episode_data["result"]["programTitle"]
title = title.replace("<", "").replace(">", "").replace("\\", "").replace("/", "").replace(":", "").replace("*", "").replace("\"", "").replace("|", "").replace("?", "").replace(" ", " ").strip()
episodeno = episode_data["result"]["episodeNo"]
airdate = episode_data["result"]["airDate"].replace('-', '')[2:]
currentQuality = episode_data["result"]["qualityList"]['qualityCurrent']
qualityRes = PooqAPI.get_quality_to_res(currentQuality)
#logger.debug('currentQuality : %s %s', currentQuality, qualityRes)
release = "SP"
if episode_data["result"]["contentType"] == 'qvod':
release = 'SPQ'
if len(episodeno):
if len(episodeno) == 1:
episodeno = "0" + episodeno
if episodeno == "특집":
ret = "%s.%s.%s.%s-%s.mp4" % (title, episodeno, airdate, qualityRes, release)
else:
ret = "%s.E%s.%s.%s-%s.mp4" % (title, episodeno, airdate, qualityRes, release)
else:
ret = "%s.%s.%s-%s.mp4" % (title, airdate, qualityRes, release)
#logger.debug('filename : %s', ret)
return ret
except Exception as exception:
logger.error('Exception:%s', exception)
logger.error(traceback.format_exc())
def download_normal(entity):
try:
url = 'https://comic.naver.com/webtoon/detail.nhn?titleId=%s&no=%s' % (
entity['title_id'], entity['episode_id'])
data = LogicNormal.get_html(url)
tree = html.fromstring(data)
entity['download_count'] += 1
entity['status'] = 1
entity['str_status'] = '대기'
LogicNormal.entity_update(entity)
entity['title'] = tree.xpath(
'//*[@id="content"]/div[1]/div[1]/div[2]/h2/span[1]/text()'
)[0].strip()
tag = tree.xpath('//*[@id="content"]/div[1]/div[2]/div[1]/h3')[0]
entity['episode_title'] = tag.text_content().strip()
logger.debug(entity['episode_title'])
entity['str_status'] = '분석'
LogicNormal.entity_update(entity)
#logger.debug(entity)
tags = tree.xpath('//*[@id="comic_view_area"]/div[1]/img')
dirname = ModelSetting.get('download_path')
if ModelSetting.get_bool('use_title_folder'):
dirname = os.path.join(
dirname,
Util.change_text_for_use_filename(entity['title']))
if not os.path.exists(dirname):
os.makedirs(dirname)
tmp = u'%s %s %s' % (entity['episode_id'].zfill(3),
entity['title'], entity['episode_title'])
if len(tmp) > 200:
tmp = u'%s %s' % (entity['episode_id'].zfill(3),
entity['title'])
dirname = os.path.join(dirname,
Util.change_text_for_use_filename(tmp))
if not os.path.exists(dirname):
os.makedirs(dirname)
entity['filename'] = '%s.zip' % tmp
if os.path.exists(entity['filename']):
entity['status'] = 12
entity['str_status'] = '파일 있음'
LogicNormal.entity_update(entity)
else:
entity['str_status'] = '다운로드중'
LogicNormal.entity_update(entity)
for idx, img in enumerate(tags):
src = img.attrib['src']
filename = os.path.join(dirname,
str(idx + 1).zfill(2) + '.jpg')
image_data = requests.get(src,
headers=headers,
stream=True)
with open(filename, 'wb') as handler:
handler.write(image_data.content)
entity['str_status'] = '다운로드중 %s / %s' % (idx + 1,
len(tags))
LogicNormal.entity_update(entity)
LogicNormal.makezip(dirname)
entity['status'] = 11
entity['str_status'] = '완료'
LogicNormal.entity_update(entity)
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
entity['status'] = 2
entity['str_status'] = '실패'
if entity['download_count'] >= 5:
entity['status'] = 13
entity['str_status'] = '재시도초과'
LogicNormal.entity_update(entity)
ModelItem.save_as_dict(entity)
def download2(self, entity):
from selenium.webdriver.support.ui import WebDriverWait
try:
from system import SystemLogicSelenium
from . import plugin
if LogicNormal.driver is None:
LogicNormal.driver = SystemLogicSelenium.create_driver()
driver = LogicNormal.driver
url = 'https://comic.naver.com/webtoon/detail.nhn?titleId=%s&no=%s' % (
entity['title_id'], entity['episode_id'])
logger.debug(url)
ret = False
driver.get(url)
entity['download_count'] += 1
entity['status'] = 1
entity['str_status'] = '대기'
LogicNormal.update_ui(self, entity)
tag = WebDriverWait(
driver, 30).until(lambda driver: driver.find_element_by_xpath(
'//*[@id="content"]/div[1]/div[1]/div[2]/h2'))
entity['title'] = SystemLogicSelenium.get_text_excluding_children(
driver, tag).strip()
tag = WebDriverWait(
driver, 30).until(lambda driver: driver.find_element_by_xpath(
'//*[@id="content"]/div[1]/div[2]/div[1]/h3'))
entity['episode_title'] = tag.text
entity['str_status'] = '분석'
LogicNormal.update_ui(self, entity)
tag = WebDriverWait(
driver, 30).until(lambda driver: driver.find_element_by_xpath(
'//*[@id="btnRemoteConOnOff"]'))
if tag.find_element_by_xpath('em').text == 'ON':
#logger.debug('ON to OFF')
tag.click()
#S = lambda X: driver.execute_script('return document.body.parentNode.scroll'+X)
#driver.set_window_size(S('Width'),S('Height')) # May need manual adjustment
#logger.debug(S('Width'))
#logger.debug(S('Height'))
tag = WebDriverWait(
driver, 30).until(lambda driver: driver.find_element_by_xpath(
'//*[@id="comic_view_area"]/div[1]'))
dirname = ModelSetting.get('download_path')
if ModelSetting.get_bool('use_title_folder'):
dirname = os.path.join(
dirname,
Util.change_text_for_use_filename(entity['title']))
if not os.path.exists(dirname):
os.makedirs(dirname)
tmp = u'%s %s %s.png' % (entity['episode_id'].zfill(3),
entity['title'], entity['episode_title'])
entity['filename'] = os.path.join(
dirname, Util.change_text_for_use_filename(tmp))
if os.path.exists(entity['filename']):
entity['status'] = 12
entity['str_status'] = '파일 있음'
LogicNormal.update_ui(self, entity)
else:
entity['str_status'] = '다운로드중'
LogicNormal.update_ui(self, entity)
from system import SystemLogicSelenium
full = SystemLogicSelenium.full_screenshot(driver)
if full is not None:
img_tag = tag.find_elements_by_xpath('img')
if len(img_tag) > 1:
img_tag = img_tag[1]
elif len(img_tag) == 1:
img_tag = img_tag[0]
else:
img_tag = tag
left = img_tag.location['x']
top = tag.location['y']
right = img_tag.location['x'] + img_tag.size['width']
bottom = tag.location['y'] + tag.size['height']
im = full.crop(
(left, top, right, bottom)) # defines crop points
im.save(entity['filename'])
entity['status'] = 11
entity['str_status'] = '완료'
LogicNormal.update_ui(self, entity)
else:
raise Exception('capture fail.')
except Exception as e:
logger.error('Exception:%s', e)
logger.error(traceback.format_exc())
entity['status'] = 2
entity['str_status'] = '실패'
if entity['download_count'] >= 5:
entity['status'] = 13
entity['str_status'] = '재시도초과'
LogicNormal.update_ui(self, entity)
ModelItem.save_as_dict(entity)
