def tag_guessing(tag_name):
tag_name = tag_name.lower()
tag_name = tag_name.replace(' ', '-')
logger.info('Trying to get tag_id of tag \'{0}\''.format(tag_name))
response = request('get', url='%s/%s' % (constant.TAG_URL, tag_name)).content
html = BeautifulSoup(response, 'html.parser')
first_item = html.find('div', attrs={'class': 'gallery'})
if not first_item:
logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
return
doujinshi_id = re.findall('(\d+)', first_item.a.attrs['href'])
if not doujinshi_id:
logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
return
ret = doujinshi_parser(doujinshi_id[0])
if 'tag' in ret and tag_name in ret['tag']:
tag_id = ret['tag'][tag_name]
logger.info('Tag id of tag \'{0}\' is {1}'.format(tag_name, tag_id))
else:
logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name))
return
return tag_id
def __api_suspended_tag_parser(tag_id, sorting, max_page=1):
logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
result = []
response = request('get',
url=constant.TAG_API_URL,
params={
'sort': sorting,
'tag_id': tag_id
}).json()
page = max_page if max_page <= response['num_pages'] else int(
response['num_pages'])
for i in range(1, page + 1):
logger.info('Getting page {} ...'.format(i))
if page != 1:
response = request('get',
url=constant.TAG_API_URL,
params={
'sort': sorting,
'tag_id': tag_id
}).json()
for row in response['result']:
title = row['title']['english']
title = title[:85] + '..' if len(title) > 85 else title
result.append({'id': row['id'], 'title': title})
if not result:
logger.warn('No results for tag id {}'.format(tag_id))
return result
def print_doujinshi(doujinshi_list):
if not doujinshi_list:
return
doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list]
headers = ['id', 'doujinshi']
logger.info('Search Result\n' +
tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
def login(username, password):
logger.warning(
'This feature is deprecated, please use --cookie to set your cookie.')
csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text)
if os.getenv('DEBUG'):
logger.info('Getting CSRF token ...')
if os.getenv('DEBUG'):
logger.info('CSRF token is {}'.format(csrf_token))
login_dict = {
'csrfmiddlewaretoken': csrf_token,
'username_or_email': username,
'password': password,
}
resp = request('post', url=constant.LOGIN_URL, data=login_dict)
if 'You\'re loading pages way too quickly.' in resp.text or 'Really, slow down' in resp.text:
csrf_token = _get_csrf_token(resp.text)
resp = request('post',
url=resp.url,
data={
'csrfmiddlewaretoken': csrf_token,
'next': '/'
})
if 'Invalid username/email or password' in resp.text:
logger.error('Login failed, please check your username and password')
exit(1)
if 'You\'re loading pages way too quickly.' in resp.text or 'Really, slow down' in resp.text:
logger.error(
'Using nhentai --cookie \'YOUR_COOKIE_HERE\' to save your Cookie.')
exit(2)
def login(username, password):
csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text)
if os.getenv('DEBUG'):
logger.info('Getting CSRF token ...')
if os.getenv('DEBUG'):
logger.info('CSRF token is {}'.format(csrf_token))
login_dict = {
'csrfmiddlewaretoken': csrf_token,
'username_or_email': username,
'password': password,
}
resp = request('post', url=constant.LOGIN_URL, data=login_dict)
if 'You\'re loading pages way too quickly.' in resp.text:
csrf_token = _get_csrf_token(resp.text)
resp = request('post', url=resp.url, data={'csrfmiddlewaretoken': csrf_token, 'next': '/'})
if 'Invalid username/email or password' in resp.text:
logger.error('Login failed, please check your username and password')
exit(1)
if 'You\'re loading pages way too quickly.' in resp.text:
logger.error('You meet challenge insistently, please submit a issue'
' at https://github.com/RicterZ/nhentai/issues')
exit(2)
def generate_metadata_file(output_dir, table, doujinshi_obj=None):
logger.info('Writing Metadata Info')
if doujinshi_obj is not None:
doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
else:
doujinshi_dir = '.'
logger.info(doujinshi_dir)
f = open(os.path.join(doujinshi_dir, 'info.txt'), 'w', encoding='utf-8')
fields = ['TITLE', 'ORIGINAL TITLE', 'AUTHOR', 'ARTIST', 'CIRCLE', 'SCANLATOR',
'TRANSLATOR', 'PUBLISHER', 'DESCRIPTION', 'STATUS', 'CHAPTERS', 'PAGES',
'TAGS', 'TYPE', 'LANGUAGE', 'RELEASED', 'READING DIRECTION', 'CHARACTERS',
'SERIES', 'PARODY', 'URL']
special_fields = ['PARODY', 'TITLE', 'ORIGINAL TITLE', 'CHARACTERS', 'AUTHOR',
'LANGUAGE', 'TAGS', 'URL', 'PAGES']
for i in range(len(fields)):
f.write('{}: '.format(fields[i]))
if fields[i] in special_fields:
f.write(str(table[special_fields.index(fields[i])][1]))
f.write('\n')
f.close()
def generate_main_html(output_dir='./'):
"""
Generate a main html to show all the contain doujinshi.
With a link to their `index.html`.
Default output folder will be the CLI path.
"""
image_html = ''
main = readfile('viewer/main.html')
css = readfile('viewer/main.css')
js = readfile('viewer/main.js')
element = '\n\
<div class="gallery-favorite">\n\
<div class="gallery">\n\
<a href="./{FOLDER}/index.html" class="cover" style="padding:0 0 141.6% 0"><img\n\
src="./{FOLDER}/{IMAGE}" />\n\
<div class="caption">{TITLE}</div>\n\
</a>\n\
</div>\n\
</div>\n'
os.chdir(output_dir)
doujinshi_dirs = next(os.walk('.'))[1]
for folder in doujinshi_dirs:
files = os.listdir(folder)
files.sort()
if 'index.html' in files:
logger.info('Add doujinshi \'{}\''.format(folder))
else:
continue
image = files[0] # 001.jpg or 001.png
if folder is not None:
title = folder.replace('_', ' ')
else:
title = 'nHentai HTML Viewer'
image_html += element.format(FOLDER=folder, IMAGE=image, TITLE=title)
if image_html == '':
logger.warning('No index.html found, --gen-main paused.')
return
try:
data = main.format(STYLES=css, SCRIPTS=js, PICTURE=image_html)
if sys.version_info < (3, 0):
with open('./main.html', 'w') as f:
f.write(data)
else:
with open('./main.html', 'wb') as f:
f.write(data.encode('utf-8'))
shutil.copy(os.path.dirname(__file__) + '/viewer/logo.png', './')
set_js_database()
logger.log(
15, 'Main Viewer has been written to \'{0}main.html\''.format(
output_dir))
except Exception as e:
logger.warning('Writing Main Viewer failed ({})'.format(str(e)))
def print_doujinshi(doujinshi_list):
if not doujinshi_list:
return
doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list]
headers = ['id', 'doujinshi']
logger.info('Search Result\n' +
tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
def download(self):
logger.info('Starting to download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
if len(self.ext) != self.pages:
logger.warning('Page count and ext count do not equal')
for i in range(1, min(self.pages, len(self.ext)) + 1):
download_queue.append(
'%s/%d/%d.%s' %
(IMAGE_URL, int(self.img_id), i, self.ext[i - 1]))
self.downloader.download(download_queue, self.filename)
with open(os.path.join(self.path, self.filename, 'ComicInfo.xml'),
"w") as f:
f.write(self.comicinfoXML)
'''
for i in range(len(self.ext)):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]]))
'''
else:
logger.critical('Downloader has not been loaded')
def login(username, password):
csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text)
if os.getenv('DEBUG'):
logger.info('Getting CSRF token ...')
if os.getenv('DEBUG'):
logger.info('CSRF token is {}'.format(csrf_token))
login_dict = {
'csrfmiddlewaretoken': csrf_token,
'username_or_email': username,
'password': password,
}
resp = request('post', url=constant.LOGIN_URL, data=login_dict)
if 'You\'re loading pages way too quickly.' in resp.text:
csrf_token = _get_csrf_token(resp.text)
resp = request('post',
url=resp.url,
data={
'csrfmiddlewaretoken': csrf_token,
'next': '/'
})
if 'Invalid username/email or password' in resp.text:
logger.error('Login failed, please check your username and password')
exit(1)
if 'You\'re loading pages way too quickly.' in resp.text:
logger.error('You meet challenge insistently, please submit a issue'
' at https://github.com/RicterZ/nhentai/issues')
exit(2)
def _download(self, url, folder='', filename='', retried=0):
logger.info('Start downloading: {0} ...'.format(url))
filename = filename if filename else os.path.basename(urlparse(url).path)
base_filename, extension = os.path.splitext(filename)
try:
with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f:
response = request('get', url, stream=True, timeout=self.timeout)
if response.status_code != 200:
raise NhentaiImageNotExistException
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(2048):
f.write(chunk)
except (requests.HTTPError, requests.Timeout) as e:
if retried < 3:
logger.warning('Warning: {0}, retrying({1}) ...'.format(str(e), retried))
return 0, self._download(url=url, folder=folder, filename=filename, retried=retried+1)
else:
return 0, None
except NhentaiImageNotExistException as e:
os.remove(os.path.join(folder, base_filename.zfill(3) + extension))
return -1, url
except Exception as e:
logger.critical(str(e))
return 0, None
return 1, url
def generate_cbz(output_dir='.',
doujinshi_obj=None,
rm_origin_dir=False,
write_comic_info=False):
if doujinshi_obj is not None:
doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
if write_comic_info:
serialize_comicxml(doujinshi_obj, doujinshi_dir)
cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'),
'{}.cbz'.format(doujinshi_obj.filename))
else:
cbz_filename = './doujinshi.cbz'
doujinshi_dir = '.'
file_list = os.listdir(doujinshi_dir)
file_list.sort()
logger.info('Writing CBZ file to path: {}'.format(cbz_filename))
with zipfile.ZipFile(cbz_filename, 'w') as cbz_pf:
for image in file_list:
image_path = os.path.join(doujinshi_dir, image)
cbz_pf.write(image_path, image)
if rm_origin_dir:
shutil.rmtree(doujinshi_dir, ignore_errors=True)
logger.log(
15, 'Comic Book CBZ file has been written to \'{0}\''.format(
doujinshi_dir))
def _download(self, url, folder='', filename='', retried=False):
logger.info('Start downloading: {0} ...'.format(url))
filename = filename if filename else os.path.basename(
urlparse(url).path)
base_filename, extension = os.path.splitext(filename)
try:
with open(os.path.join(folder,
base_filename.zfill(3) + extension),
"wb") as f:
response = request('get',
url,
stream=True,
timeout=self.timeout)
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(2048):
f.write(chunk)
except requests.HTTPError as e:
if not retried:
logger.error('Error: {0}, retrying'.format(str(e)))
return self._download(url=url,
folder=folder,
filename=filename,
retried=True)
else:
return None
except Exception as e:
logger.critical(str(e))
return None
return url
def generate_pdf(output_dir='.', doujinshi_obj=None, rm_origin_dir=False):
"""Write images to a PDF file using img2pdf."""
if doujinshi_obj is not None:
doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
pdf_filename = os.path.join(os.path.join(doujinshi_dir, '..'),
'{}.pdf'.format(doujinshi_obj.filename))
else:
pdf_filename = './doujinshi.pdf'
doujinshi_dir = '.'
file_list = os.listdir(doujinshi_dir)
file_list.sort()
logger.info('Writing PDF file to path: {}'.format(pdf_filename))
with open(pdf_filename, 'wb') as pdf_f:
full_path_list = ([
os.path.join(doujinshi_dir, image) for image in file_list
])
pdf_f.write(img2pdf.convert(full_path_list))
if rm_origin_dir:
shutil.rmtree(doujinshi_dir, ignore_errors=True)
logger.log(15,
'PDF file has been written to \'{0}\''.format(doujinshi_dir))
def check_cookie():
response = request('get', constant.BASE_URL).text
username = re.findall('"/users/\d+/(.*?)"', response)
if not username:
logger.error('Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie')
else:
logger.info('Login successfully! Your username: {}'.format(username[0]))
def _download(self, url, folder='', filename='', retried=0):
logger.info('Starting to download {0} ...'.format(url))
filename = filename if filename else os.path.basename(
urlparse(url).path)
base_filename, extension = os.path.splitext(filename)
try:
if os.path.exists(
os.path.join(folder,
base_filename.zfill(3) + extension)):
logger.warning('File: {0} exists, ignoring'.format(
os.path.join(folder,
base_filename.zfill(3) + extension)))
return 1, url
with open(os.path.join(folder,
base_filename.zfill(3) + extension),
"wb") as f:
i = 0
while i < 10:
try:
response = request('get',
url,
stream=True,
timeout=self.timeout)
except Exception as e:
i += 1
if not i < 10:
logger.critical(str(e))
return 0, None
continue
break
if response.status_code != 200:
raise NhentaiImageNotExistException
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(2048):
f.write(chunk)
except (requests.HTTPError, requests.Timeout) as e:
if retried < 3:
logger.warning('Warning: {0}, retrying({1}) ...'.format(
str(e), retried))
return 0, self._download(url=url,
folder=folder,
filename=filename,
retried=retried + 1)
else:
return 0, None
except NhentaiImageNotExistException as e:
os.remove(os.path.join(folder, base_filename.zfill(3) + extension))
return -1, url
except Exception as e:
logger.critical(str(e))
return 0, None
return 1, url
def main():
banner()
logger.info('Using mirror: {0}'.format(BASE_URL))
options = cmd_parser()
doujinshi_ids = []
doujinshi_list = []
if options.login:
username, password = options.login.split(':', 1)
logger.info('Logging in to nhentai using credential pair \'%s:%s\'' %
(username, '*' * len(password)))
login(username, password)
if options.is_download:
for doujinshi_info in login_parser():
doujinshi_list.append(Doujinshi(**doujinshi_info))
if options.tag:
doujinshis = tag_parser(options.tag, max_page=options.max_page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
if options.keyword:
doujinshis = search_parser(options.keyword, options.page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
if not doujinshi_ids:
doujinshi_ids = options.id
if doujinshi_ids:
for id_ in doujinshi_ids:
doujinshi_info = doujinshi_parser(id_)
doujinshi_list.append(Doujinshi(**doujinshi_info))
if not options.is_show:
downloader = Downloader(path=options.output_dir,
thread=options.threads,
timeout=options.timeout)
for doujinshi in doujinshi_list:
doujinshi.downloader = downloader
doujinshi.download()
if not options.is_nohtml and not options.is_cbz:
generate_html(options.output_dir, doujinshi)
elif options.is_cbz:
generate_cbz(options.output_dir, doujinshi,
options.rm_origin_dir)
if not platform.system() == 'Windows':
logger.log(15, '🍻 All done.')
else:
logger.log(15, 'All done.')
else:
[doujinshi.show() for doujinshi in doujinshi_list]
def banner():
logger.info(u'''nHentai: あなたも変態。 いいね?
_ _ _ _
_ __ | | | | ___ _ __ | |_ __ _(_)
| '_ \| |_| |/ _ \ '_ \| __/ _` | |
| | | | _ | __/ | | | || (_| | |
|_| |_|_| |_|\___|_| |_|\__\__,_|_|
''')
def banner():
logger.info(u'''nHentai ver %s: あなたも変態。 いいね?
_ _ _ _
_ __ | | | | ___ _ __ | |_ __ _(_)
| '_ \| |_| |/ _ \ '_ \| __/ _` | |
| | | | _ | __/ | | | || (_| | |
|_| |_|_| |_|\___|_| |_|\__\__,_|_|
''' % __version__)
def banner():
logger.info('''nHentai: あなたも変態。 いいね?
_ _ _ _
_ __ | | | | ___ _ __ | |_ __ _(_)
| '_ \| |_| |/ _ \ '_ \| __/ _` | |
| | | | _ | __/ | | | || (_| | |
|_| |_|_| |_|\___|_| |_|\__\__,_|_|
''')
def login_parser(username, password):
s = requests.Session()
s.proxies = constant.PROXY
s.verify = False
s.headers.update({'Referer': constant.LOGIN_URL})
s.get(constant.LOGIN_URL)
content = s.get(constant.LOGIN_URL).content
html = BeautifulSoup(content, 'html.parser')
csrf_token_elem = html.find('input', attrs={'name': 'csrfmiddlewaretoken'})
if not csrf_token_elem:
raise Exception('Cannot find csrf token to login')
csrf_token = csrf_token_elem.attrs['value']
login_dict = {
'csrfmiddlewaretoken': csrf_token,
'username_or_email': username,
'password': password,
}
resp = s.post(constant.LOGIN_URL, data=login_dict)
if 'Invalid username (or email) or password' in resp.text:
logger.error('Login failed, please check your username and password')
exit(1)
html = BeautifulSoup(s.get(constant.FAV_URL).content, 'html.parser')
count = html.find('span', attrs={'class': 'count'})
if not count:
logger.error('Cannot get count of your favorites, maybe login failed.')
count = int(count.text.strip('(').strip(')'))
pages = count / 25
pages += 1 if count % (25 * pages) else 0
logger.info('Your have %d favorites in %d pages.' % (count, pages))
if os.getenv('DEBUG'):
pages = 1
ret = []
doujinshi_id = re.compile('data-id="([\d]+)"')
def _callback(request, result):
ret.append(result)
thread_pool = threadpool.ThreadPool(5)
for page in range(1, pages+1):
try:
logger.info('Getting doujinshi id of page %d' % page)
resp = s.get(constant.FAV_URL + '?page=%d' % page).content
ids = doujinshi_id.findall(resp)
requests_ = threadpool.makeRequests(doujinshi_parser, ids, _callback)
[thread_pool.putRequest(req) for req in requests_]
thread_pool.wait()
except Exception as e:
logger.error('Error: %s, continue', str(e))
return ret
def download(self):
logger.info('Start download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
for i in range(1, self.pages + 1):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
self.downloader.download(download_queue, format_filename('%s-%s' % (self.id, self.name[:200])))
else:
logger.critical('Downloader has not be loaded')
def download(self):
logger.info('Start download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
for i in range(1, self.pages + 1):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext))
self.downloader.download(download_queue, self.id)
else:
logger.critical('Downloader has not be loaded')
def main():
banner()
logger.info('Using mirror: {0}'.format(BASE_URL))
options = cmd_parser()
doujinshi_ids = []
doujinshi_list = []
if options.login:
username, password = options.login.split(':', 1)
logger.info('Logging in to nhentai using credential pair \'%s:%s\'' % (username, '*' * len(password)))
login(username, password)
if options.is_download:
for doujinshi_info in login_parser():
doujinshi_list.append(Doujinshi(**doujinshi_info))
if options.tag:
doujinshis = tag_parser(options.tag, max_page=options.max_page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
if options.keyword:
doujinshis = search_parser(options.keyword, options.page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
if not doujinshi_ids:
doujinshi_ids = options.id
if doujinshi_ids:
for id_ in doujinshi_ids:
doujinshi_info = doujinshi_parser(id_)
doujinshi_list.append(Doujinshi(**doujinshi_info))
if not options.is_show:
downloader = Downloader(path=options.output_dir,
thread=options.threads, timeout=options.timeout)
for doujinshi in doujinshi_list:
doujinshi.downloader = downloader
doujinshi.download()
if not options.is_nohtml and not options.is_cbz:
generate_html(options.output_dir, doujinshi)
elif options.is_cbz:
generate_cbz(options.output_dir, doujinshi, options.rm_origin_dir)
if not platform.system() == 'Windows':
logger.log(15, '🍻 All done.')
else:
logger.log(15, 'All done.')
else:
[doujinshi.show() for doujinshi in doujinshi_list]
def search_parser(keyword, sorting, page, is_page_all=False):
# keyword = '+'.join([i.strip().replace(' ', '-').lower() for i in keyword.split(',')])
result = []
response = None
if not page:
page = [1]
if is_page_all:
url = request('get',
url=constant.SEARCH_URL,
params={
'query': keyword
}).url
init_response = request('get', url.replace('%2B', '+')).json()
page = range(1, init_response['num_pages'] + 1)
total = '/{0}'.format(page[-1]) if is_page_all else ''
not_exists_persist = False
for p in page:
i = 0
logger.info(
'Searching doujinshis using keywords "{0}" on page {1}{2}'.format(
keyword, p, total))
while i < 3:
try:
url = request('get',
url=constant.SEARCH_URL,
params={
'query': keyword,
'page': p,
'sort': sorting
}).url
response = request('get', url.replace('%2B', '+')).json()
except Exception as e:
logger.critical(str(e))
response = None
break
if response is None or 'result' not in response:
logger.warning('No result in response in page {}'.format(p))
if not_exists_persist is True:
break
continue
for row in response['result']:
title = row['title']['english']
title = title[:85] + '..' if len(title) > 85 else title
result.append({'id': row['id'], 'title': title})
not_exists_persist = False
if not result:
logger.warning('No results for keywords {}'.format(keyword))
return result
def main():
banner()
logger.info('Using mirror: {0}'.format(BASE_URL))
options = cmd_parser()
doujinshi_ids = []
doujinshi_list = []
if options.favorites:
if not options.is_download:
logger.warning('You do not specify --download option')
doujinshi_ids = favorites_parser()
elif options.tag:
doujinshis = tag_parser(options.tag, max_page=options.max_page)
print_doujinshi(doujinshis)
if options.is_download and doujinshis:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
elif options.keyword:
doujinshis = search_parser(options.keyword, options.page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
elif not doujinshi_ids:
doujinshi_ids = options.id
if doujinshi_ids:
for id_ in doujinshi_ids:
if options.delay:
time.sleep(options.delay)
doujinshi_info = doujinshi_parser(id_)
doujinshi_list.append(Doujinshi(name_format=options.name_format, path=options.output_dir, **doujinshi_info))
if not options.is_show:
downloader = Downloader(path=options.output_dir,
thread=options.threads, timeout=options.timeout, delay=options.delay)
for doujinshi in doujinshi_list:
doujinshi.downloader = downloader
doujinshi.download()
if not options.is_nohtml and not options.is_cbz:
generate_html(options.output_dir, doujinshi)
elif options.is_cbz:
generate_cbz(options.output_dir, doujinshi, options.rm_origin_dir)
if not platform.system() == 'Windows':
logger.log(15, '🍻 All done.')
else:
logger.log(15, 'All done.')
else:
[doujinshi.show() for doujinshi in doujinshi_list]
def show(self):
table = [
["Doujinshi", self.name],
["Subtitle", self.info.subtitle],
["Characters", self.info.character],
["Authors", self.info.artist],
["Language", self.info.language],
["Tags", ', '.join(self.info.tag.keys())],
["URL", self.url],
["Pages", self.pages],
]
logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(table)))
def show(self):
table = [
["Doujinshi", self.name],
["Subtitle", self.info.subtitle],
["Characters", self.info.characters],
["Authors", self.info.artists],
["Language", self.info.language],
["Tags", self.info.tags],
["URL", self.url],
["Pages", self.pages],
]
logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(table)))
def generate_metadatafile(output_dir, table, doujinshi_obj=None):
logger.info("Writing Metadata Info")
if doujinshi_obj is not None:
doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
else:
doujinshi_dir = '.'
logger.info(doujinshi_dir)
f = open(os.path.join(doujinshi_dir, 'info.txt'), "w", encoding="utf-8")
fields = [
"TITLE", "ORIGINAL TITLE", "AUTHOR", "ARTIST", "CIRCLE", "SCANLATOR",
"TRANSLATOR", "PUBLISHER", "DESCRIPTION", "STATUS", "CHAPTERS",
"PAGES", "TAGS", "TYPE", "LANGUAGE", "RELEASED", "READING DIRECTION",
"CHARACTERS", "SERIES", "PARODY", "URL"
]
for i in range(21):
f.write("%s: " % fields[i])
if (i == 19):
f.write("%s" % table[0][1])
if (i == 0):
f.write("%s" % table[1][1])
if (i == 1):
f.write("%s" % table[2][1])
if (i == 17):
f.write("%s" % table[3][1])
if (i == 2):
f.write("%s" % table[4][1])
if (i == 14):
f.write("%s" % table[5][1])
if (i == 12):
f.write("%s" % table[6][1])
if (i == 20):
f.write("%s" % table[7][1])
if (i == 11):
f.write("%s" % table[8][1])
f.write("\n")
f.close()
def download(self):
logger.info('Starting to download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
if len(self.ext) != self.pages:
logger.warning('Page count and ext count do not equal')
for i in range(1, min(self.pages, len(self.ext)) + 1):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext[i - 1]))
self.downloader.download(download_queue, self.filename)
else:
logger.critical('Downloader has not been loaded')
def tag_parser(tag_id, max_page=1):
logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
result = []
i = 0
while i < 5:
try:
response = request('get',
url=constant.TAG_API_URL,
params={
'sort': 'popular',
'tag_id': tag_id
}).json()
except Exception as e:
i += 1
if not i < 5:
logger.critical(str(e))
exit(1)
continue
break
page = max_page if max_page <= response['num_pages'] else int(
response['num_pages'])
for i in range(1, page + 1):
logger.info('Getting page {} ...'.format(i))
if page != 1:
i = 0
while i < 5:
try:
response = request('get',
url=constant.TAG_API_URL,
params={
'sort': 'popular',
'tag_id': tag_id
}).json()
except Exception as e:
i += 1
if not i < 5:
logger.critical(str(e))
exit(1)
continue
break
for row in response['result']:
title = row['title']['english']
title = title[:85] + '..' if len(title) > 85 else title
result.append({'id': row['id'], 'title': title})
if not result:
logger.warn('No results for tag id {}'.format(tag_id))
return result
def tag_parser(tag_id):
logger.info('Get doujinshi of tag id: {0}'.format(tag_id))
result = []
response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
for row in response['result']:
title = row['title']['english']
title = title[:85] + '..' if len(title) > 85 else title
result.append({'id': row['id'], 'title': title})
if not result:
logger.warn('Not found anything of tag id {}'.format(tag_id))
return result
def check_cookie():
response = request('get', constant.BASE_URL)
if response.status_code == 503 and 'cf-browser-verification' in response.text:
logger.error(
'Blocked by Cloudflare captcha, please set your cookie and useragent'
)
exit(-1)
username = re.findall('"/users/\d+/(.*?)"', response.text)
if not username:
logger.warning(
'Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie'
)
else:
logger.info('Login successfully! Your username: {}'.format(
username[0]))
def login_parser():
html = BeautifulSoup(
request('get', constant.FAV_URL).content, 'html.parser')
count = html.find('span', attrs={'class': 'count'})
if not count:
logger.error(
"Can't get your number of favorited doujins. Did the login failed?"
)
return
count = int(count.text.strip('(').strip(')').replace(',', ''))
if count == 0:
logger.warning('No favorites found')
return []
pages = int(count / 25)
if pages:
pages += 1 if count % (25 * pages) else 0
else:
pages = 1
logger.info('You have %d favorites in %d pages.' % (count, pages))
if os.getenv('DEBUG'):
pages = 1
ret = []
doujinshi_id = re.compile('data-id="([\d]+)"')
def _callback(request, result):
ret.append(result)
thread_pool = threadpool.ThreadPool(5)
for page in range(1, pages + 1):
try:
logger.info('Getting doujinshi ids of page %d' % page)
resp = request('get', constant.FAV_URL + '?page=%d' % page).text
ids = doujinshi_id.findall(resp)
requests_ = threadpool.makeRequests(doujinshi_parser, ids,
_callback)
[thread_pool.putRequest(req) for req in requests_]
thread_pool.wait()
except Exception as e:
logger.error('Error: %s, continue', str(e))
return ret
def login_parser():
html = BeautifulSoup(request('get', constant.FAV_URL).content, 'html.parser')
count = html.find('span', attrs={'class': 'count'})
if not count:
logger.error("Can't get your number of favorited doujins. Did the login failed?")
return
count = int(count.text.strip('(').strip(')').replace(',', ''))
if count == 0:
logger.warning('No favorites found')
return []
pages = int(count / 25)
if pages:
pages += 1 if count % (25 * pages) else 0
else:
pages = 1
logger.info('You have %d favorites in %d pages.' % (count, pages))
if os.getenv('DEBUG'):
pages = 1
ret = []
doujinshi_id = re.compile('data-id="([\d]+)"')
def _callback(request, result):
ret.append(result)
thread_pool = threadpool.ThreadPool(5)
for page in range(1, pages + 1):
try:
logger.info('Getting doujinshi ids of page %d' % page)
resp = request('get', constant.FAV_URL + '?page=%d' % page).text
ids = doujinshi_id.findall(resp)
requests_ = threadpool.makeRequests(doujinshi_parser, ids, _callback)
[thread_pool.putRequest(req) for req in requests_]
thread_pool.wait()
except Exception as e:
logger.error('Error: %s, continue', str(e))
return ret
def favorites_parser(page=None):
result = []
html = BeautifulSoup(
request('get', constant.FAV_URL).content, 'html.parser')
count = html.find('span', attrs={'class': 'count'})
if not count:
logger.error(
"Can't get your number of favorited doujins. Did the login failed?"
)
return []
count = int(count.text.strip('(').strip(')').replace(',', ''))
if count == 0:
logger.warning('No favorites found')
return []
pages = int(count / 25)
if page:
page_range_list = page
else:
if pages:
pages += 1 if count % (25 * pages) else 0
else:
pages = 1
logger.info('You have %d favorites in %d pages.' % (count, pages))
if os.getenv('DEBUG'):
pages = 1
page_range_list = range(1, pages + 1)
for page in page_range_list:
try:
logger.info('Getting doujinshi ids of page %d' % page)
resp = request('get', constant.FAV_URL + '?page=%d' % page).content
result.extend(_get_title_and_id(resp))
except Exception as e:
logger.error('Error: %s, continue', str(e))
return result
def download(self):
logger.info('Starting to download doujinshi: %s' % self.name)
if self.downloader:
download_queue = []
if len(self.ext) != self.pages:
logger.warning('Page count and ext count do not equal')
for i in range(1, min(self.pages, len(self.ext)) + 1):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext[i-1]))
self.downloader.download(download_queue, self.filename)
'''
for i in range(len(self.ext)):
download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]]))
'''
else:
logger.critical('Downloader has not been loaded')
def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False):
if doujinshi_obj is not None:
doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename)
cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '%s.cbz' % doujinshi_obj.id)
else:
cbz_filename = './doujinshi.cbz'
doujinshi_dir = '.'
file_list = os.listdir(doujinshi_dir)
file_list.sort()
logger.info('Writing CBZ file to path: {}'.format(cbz_filename))
with zipfile.ZipFile(cbz_filename, 'w') as cbz_pf:
for image in file_list:
image_path = os.path.join(doujinshi_dir, image)
cbz_pf.write(image_path, image)
if rm_origin_dir:
shutil.rmtree(doujinshi_dir, ignore_errors=True)
logger.log(15, 'Comic Book CBZ file has been write to \'{0}\''.format(doujinshi_dir))
def __api_suspended_tag_parser(tag_id, max_page=1):
logger.info('Searching for doujinshi with tag id {0}'.format(tag_id))
result = []
response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json()
page = max_page if max_page <= response['num_pages'] else int(response['num_pages'])
for i in range(1, page + 1):
logger.info('Getting page {} ...'.format(i))
if page != 1:
response = request('get', url=constant.TAG_API_URL,
params={'sort': 'popular', 'tag_id': tag_id}).json()
for row in response['result']:
title = row['title']['english']
title = title[:85] + '..' if len(title) > 85 else title
result.append({'id': row['id'], 'title': title})
if not result:
logger.warn('No results for tag id {}'.format(tag_id))
return result
def main():
banner()
logger.info('Using mirror: {0}'.format(BASE_URL))
options = cmd_parser()
doujinshi_ids = []
doujinshi_list = []
if options.keyword:
doujinshis = search_parser(options.keyword, options.page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
else:
doujinshi_ids = options.id
if doujinshi_ids:
for id in doujinshi_ids:
doujinshi_info = doujinshi_parser(id)
doujinshi_list.append(Doujinshi(**doujinshi_info))
else:
exit(0)
if not options.is_show:
downloader = Downloader(path=options.output_dir,
thread=options.threads, timeout=options.timeout)
for doujinshi in doujinshi_list:
doujinshi.downloader = downloader
doujinshi.download()
generate_html(options.output_dir, doujinshi)
if not platform.system() == 'Windows':
logger.log(15, '🍺 All done.')
else:
logger.log(15, 'All done.')
else:
[doujinshi.show() for doujinshi in doujinshi_list]
def main():
banner()
logger.info('Using mirror: {0}'.format(BASE_URL))
options = cmd_parser()
doujinshi_ids = []
doujinshi_list = []
if options.keyword:
doujinshis = search_parser(options.keyword, options.page)
print_doujinshi(doujinshis)
if options.is_download:
doujinshi_ids = map(lambda d: d['id'], doujinshis)
else:
doujinshi_ids = options.id
if doujinshi_ids:
for id in doujinshi_ids:
doujinshi_info = doujinshi_parser(id)
doujinshi_list.append(Doujinshi(**doujinshi_info))
else:
exit(0)
if not options.is_show:
downloader = Downloader(path=options.output_dir,
thread=options.threads, timeout=options.timeout)
for doujinshi in doujinshi_list:
doujinshi.downloader = downloader
doujinshi.download()
if not platform.system() == 'Windows':
logger.log(15, '🍺 All done.')
else:
logger.log(15, 'All done.')
else:
[doujinshi.show() for doujinshi in doujinshi_list]
def show(self):
logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(self.table)))