def tag_guessing(tag_name): tag_name = tag_name.lower() tag_name = tag_name.replace(' ', '-') logger.info('Trying to get tag_id of tag \'{0}\''.format(tag_name)) response = request('get', url='%s/%s' % (constant.TAG_URL, tag_name)).content html = BeautifulSoup(response, 'html.parser') first_item = html.find('div', attrs={'class': 'gallery'}) if not first_item: logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name)) return doujinshi_id = re.findall('(\d+)', first_item.a.attrs['href']) if not doujinshi_id: logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name)) return ret = doujinshi_parser(doujinshi_id[0]) if 'tag' in ret and tag_name in ret['tag']: tag_id = ret['tag'][tag_name] logger.info('Tag id of tag \'{0}\' is {1}'.format(tag_name, tag_id)) else: logger.error('Cannot find doujinshi id of tag \'{0}\''.format(tag_name)) return return tag_id
def __api_suspended_tag_parser(tag_id, sorting, max_page=1): logger.info('Searching for doujinshi with tag id {0}'.format(tag_id)) result = [] response = request('get', url=constant.TAG_API_URL, params={ 'sort': sorting, 'tag_id': tag_id }).json() page = max_page if max_page <= response['num_pages'] else int( response['num_pages']) for i in range(1, page + 1): logger.info('Getting page {} ...'.format(i)) if page != 1: response = request('get', url=constant.TAG_API_URL, params={ 'sort': sorting, 'tag_id': tag_id }).json() for row in response['result']: title = row['title']['english'] title = title[:85] + '..' if len(title) > 85 else title result.append({'id': row['id'], 'title': title}) if not result: logger.warn('No results for tag id {}'.format(tag_id)) return result
def print_doujinshi(doujinshi_list): if not doujinshi_list: return doujinshi_list = [(i['id'], i['title']) for i in doujinshi_list] headers = ['id', 'doujinshi'] logger.info('Search Result\n' + tabulate(tabular_data=doujinshi_list, headers=headers, tablefmt='rst'))
def login(username, password): logger.warning( 'This feature is deprecated, please use --cookie to set your cookie.') csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text) if os.getenv('DEBUG'): logger.info('Getting CSRF token ...') if os.getenv('DEBUG'): logger.info('CSRF token is {}'.format(csrf_token)) login_dict = { 'csrfmiddlewaretoken': csrf_token, 'username_or_email': username, 'password': password, } resp = request('post', url=constant.LOGIN_URL, data=login_dict) if 'You\'re loading pages way too quickly.' in resp.text or 'Really, slow down' in resp.text: csrf_token = _get_csrf_token(resp.text) resp = request('post', url=resp.url, data={ 'csrfmiddlewaretoken': csrf_token, 'next': '/' }) if 'Invalid username/email or password' in resp.text: logger.error('Login failed, please check your username and password') exit(1) if 'You\'re loading pages way too quickly.' in resp.text or 'Really, slow down' in resp.text: logger.error( 'Using nhentai --cookie \'YOUR_COOKIE_HERE\' to save your Cookie.') exit(2)
def login(username, password): csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text) if os.getenv('DEBUG'): logger.info('Getting CSRF token ...') if os.getenv('DEBUG'): logger.info('CSRF token is {}'.format(csrf_token)) login_dict = { 'csrfmiddlewaretoken': csrf_token, 'username_or_email': username, 'password': password, } resp = request('post', url=constant.LOGIN_URL, data=login_dict) if 'You\'re loading pages way too quickly.' in resp.text: csrf_token = _get_csrf_token(resp.text) resp = request('post', url=resp.url, data={'csrfmiddlewaretoken': csrf_token, 'next': '/'}) if 'Invalid username/email or password' in resp.text: logger.error('Login failed, please check your username and password') exit(1) if 'You\'re loading pages way too quickly.' in resp.text: logger.error('You meet challenge insistently, please submit a issue' ' at https://github.com/RicterZ/nhentai/issues') exit(2)
def generate_metadata_file(output_dir, table, doujinshi_obj=None): logger.info('Writing Metadata Info') if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) else: doujinshi_dir = '.' logger.info(doujinshi_dir) f = open(os.path.join(doujinshi_dir, 'info.txt'), 'w', encoding='utf-8') fields = ['TITLE', 'ORIGINAL TITLE', 'AUTHOR', 'ARTIST', 'CIRCLE', 'SCANLATOR', 'TRANSLATOR', 'PUBLISHER', 'DESCRIPTION', 'STATUS', 'CHAPTERS', 'PAGES', 'TAGS', 'TYPE', 'LANGUAGE', 'RELEASED', 'READING DIRECTION', 'CHARACTERS', 'SERIES', 'PARODY', 'URL'] special_fields = ['PARODY', 'TITLE', 'ORIGINAL TITLE', 'CHARACTERS', 'AUTHOR', 'LANGUAGE', 'TAGS', 'URL', 'PAGES'] for i in range(len(fields)): f.write('{}: '.format(fields[i])) if fields[i] in special_fields: f.write(str(table[special_fields.index(fields[i])][1])) f.write('\n') f.close()
def generate_main_html(output_dir='./'): """ Generate a main html to show all the contain doujinshi. With a link to their `index.html`. Default output folder will be the CLI path. """ image_html = '' main = readfile('viewer/main.html') css = readfile('viewer/main.css') js = readfile('viewer/main.js') element = '\n\ <div class="gallery-favorite">\n\ <div class="gallery">\n\ <a href="./{FOLDER}/index.html" class="cover" style="padding:0 0 141.6% 0"><img\n\ src="./{FOLDER}/{IMAGE}" />\n\ <div class="caption">{TITLE}</div>\n\ </a>\n\ </div>\n\ </div>\n' os.chdir(output_dir) doujinshi_dirs = next(os.walk('.'))[1] for folder in doujinshi_dirs: files = os.listdir(folder) files.sort() if 'index.html' in files: logger.info('Add doujinshi \'{}\''.format(folder)) else: continue image = files[0] # 001.jpg or 001.png if folder is not None: title = folder.replace('_', ' ') else: title = 'nHentai HTML Viewer' image_html += element.format(FOLDER=folder, IMAGE=image, TITLE=title) if image_html == '': logger.warning('No index.html found, --gen-main paused.') return try: data = main.format(STYLES=css, SCRIPTS=js, PICTURE=image_html) if sys.version_info < (3, 0): with open('./main.html', 'w') as f: f.write(data) else: with open('./main.html', 'wb') as f: f.write(data.encode('utf-8')) shutil.copy(os.path.dirname(__file__) + '/viewer/logo.png', './') set_js_database() logger.log( 15, 'Main Viewer has been written to \'{0}main.html\''.format( output_dir)) except Exception as e: logger.warning('Writing Main Viewer failed ({})'.format(str(e)))
def download(self): logger.info('Starting to download doujinshi: %s' % self.name) if self.downloader: download_queue = [] if len(self.ext) != self.pages: logger.warning('Page count and ext count do not equal') for i in range(1, min(self.pages, len(self.ext)) + 1): download_queue.append( '%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext[i - 1])) self.downloader.download(download_queue, self.filename) with open(os.path.join(self.path, self.filename, 'ComicInfo.xml'), "w") as f: f.write(self.comicinfoXML) ''' for i in range(len(self.ext)): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]])) ''' else: logger.critical('Downloader has not been loaded')
def login(username, password): csrf_token = _get_csrf_token(request('get', url=constant.LOGIN_URL).text) if os.getenv('DEBUG'): logger.info('Getting CSRF token ...') if os.getenv('DEBUG'): logger.info('CSRF token is {}'.format(csrf_token)) login_dict = { 'csrfmiddlewaretoken': csrf_token, 'username_or_email': username, 'password': password, } resp = request('post', url=constant.LOGIN_URL, data=login_dict) if 'You\'re loading pages way too quickly.' in resp.text: csrf_token = _get_csrf_token(resp.text) resp = request('post', url=resp.url, data={ 'csrfmiddlewaretoken': csrf_token, 'next': '/' }) if 'Invalid username/email or password' in resp.text: logger.error('Login failed, please check your username and password') exit(1) if 'You\'re loading pages way too quickly.' in resp.text: logger.error('You meet challenge insistently, please submit a issue' ' at https://github.com/RicterZ/nhentai/issues') exit(2)
def _download(self, url, folder='', filename='', retried=0): logger.info('Start downloading: {0} ...'.format(url)) filename = filename if filename else os.path.basename(urlparse(url).path) base_filename, extension = os.path.splitext(filename) try: with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f: response = request('get', url, stream=True, timeout=self.timeout) if response.status_code != 200: raise NhentaiImageNotExistException length = response.headers.get('content-length') if length is None: f.write(response.content) else: for chunk in response.iter_content(2048): f.write(chunk) except (requests.HTTPError, requests.Timeout) as e: if retried < 3: logger.warning('Warning: {0}, retrying({1}) ...'.format(str(e), retried)) return 0, self._download(url=url, folder=folder, filename=filename, retried=retried+1) else: return 0, None except NhentaiImageNotExistException as e: os.remove(os.path.join(folder, base_filename.zfill(3) + extension)) return -1, url except Exception as e: logger.critical(str(e)) return 0, None return 1, url
def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False, write_comic_info=False): if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) if write_comic_info: serialize_comicxml(doujinshi_obj, doujinshi_dir) cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '{}.cbz'.format(doujinshi_obj.filename)) else: cbz_filename = './doujinshi.cbz' doujinshi_dir = '.' file_list = os.listdir(doujinshi_dir) file_list.sort() logger.info('Writing CBZ file to path: {}'.format(cbz_filename)) with zipfile.ZipFile(cbz_filename, 'w') as cbz_pf: for image in file_list: image_path = os.path.join(doujinshi_dir, image) cbz_pf.write(image_path, image) if rm_origin_dir: shutil.rmtree(doujinshi_dir, ignore_errors=True) logger.log( 15, 'Comic Book CBZ file has been written to \'{0}\''.format( doujinshi_dir))
def _download(self, url, folder='', filename='', retried=False): logger.info('Start downloading: {0} ...'.format(url)) filename = filename if filename else os.path.basename( urlparse(url).path) base_filename, extension = os.path.splitext(filename) try: with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f: response = request('get', url, stream=True, timeout=self.timeout) length = response.headers.get('content-length') if length is None: f.write(response.content) else: for chunk in response.iter_content(2048): f.write(chunk) except requests.HTTPError as e: if not retried: logger.error('Error: {0}, retrying'.format(str(e))) return self._download(url=url, folder=folder, filename=filename, retried=True) else: return None except Exception as e: logger.critical(str(e)) return None return url
def generate_pdf(output_dir='.', doujinshi_obj=None, rm_origin_dir=False): """Write images to a PDF file using img2pdf.""" if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) pdf_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '{}.pdf'.format(doujinshi_obj.filename)) else: pdf_filename = './doujinshi.pdf' doujinshi_dir = '.' file_list = os.listdir(doujinshi_dir) file_list.sort() logger.info('Writing PDF file to path: {}'.format(pdf_filename)) with open(pdf_filename, 'wb') as pdf_f: full_path_list = ([ os.path.join(doujinshi_dir, image) for image in file_list ]) pdf_f.write(img2pdf.convert(full_path_list)) if rm_origin_dir: shutil.rmtree(doujinshi_dir, ignore_errors=True) logger.log(15, 'PDF file has been written to \'{0}\''.format(doujinshi_dir))
def check_cookie(): response = request('get', constant.BASE_URL).text username = re.findall('"/users/\d+/(.*?)"', response) if not username: logger.error('Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie') else: logger.info('Login successfully! Your username: {}'.format(username[0]))
def _download(self, url, folder='', filename='', retried=0): logger.info('Starting to download {0} ...'.format(url)) filename = filename if filename else os.path.basename( urlparse(url).path) base_filename, extension = os.path.splitext(filename) try: if os.path.exists( os.path.join(folder, base_filename.zfill(3) + extension)): logger.warning('File: {0} exists, ignoring'.format( os.path.join(folder, base_filename.zfill(3) + extension))) return 1, url with open(os.path.join(folder, base_filename.zfill(3) + extension), "wb") as f: i = 0 while i < 10: try: response = request('get', url, stream=True, timeout=self.timeout) except Exception as e: i += 1 if not i < 10: logger.critical(str(e)) return 0, None continue break if response.status_code != 200: raise NhentaiImageNotExistException length = response.headers.get('content-length') if length is None: f.write(response.content) else: for chunk in response.iter_content(2048): f.write(chunk) except (requests.HTTPError, requests.Timeout) as e: if retried < 3: logger.warning('Warning: {0}, retrying({1}) ...'.format( str(e), retried)) return 0, self._download(url=url, folder=folder, filename=filename, retried=retried + 1) else: return 0, None except NhentaiImageNotExistException as e: os.remove(os.path.join(folder, base_filename.zfill(3) + extension)) return -1, url except Exception as e: logger.critical(str(e)) return 0, None return 1, url
def main(): banner() logger.info('Using mirror: {0}'.format(BASE_URL)) options = cmd_parser() doujinshi_ids = [] doujinshi_list = [] if options.login: username, password = options.login.split(':', 1) logger.info('Logging in to nhentai using credential pair \'%s:%s\'' % (username, '*' * len(password))) login(username, password) if options.is_download: for doujinshi_info in login_parser(): doujinshi_list.append(Doujinshi(**doujinshi_info)) if options.tag: doujinshis = tag_parser(options.tag, max_page=options.max_page) print_doujinshi(doujinshis) if options.is_download: doujinshi_ids = map(lambda d: d['id'], doujinshis) if options.keyword: doujinshis = search_parser(options.keyword, options.page) print_doujinshi(doujinshis) if options.is_download: doujinshi_ids = map(lambda d: d['id'], doujinshis) if not doujinshi_ids: doujinshi_ids = options.id if doujinshi_ids: for id_ in doujinshi_ids: doujinshi_info = doujinshi_parser(id_) doujinshi_list.append(Doujinshi(**doujinshi_info)) if not options.is_show: downloader = Downloader(path=options.output_dir, thread=options.threads, timeout=options.timeout) for doujinshi in doujinshi_list: doujinshi.downloader = downloader doujinshi.download() if not options.is_nohtml and not options.is_cbz: generate_html(options.output_dir, doujinshi) elif options.is_cbz: generate_cbz(options.output_dir, doujinshi, options.rm_origin_dir) if not platform.system() == 'Windows': logger.log(15, '🍻 All done.') else: logger.log(15, 'All done.') else: [doujinshi.show() for doujinshi in doujinshi_list]
def banner(): logger.info(u'''nHentai: あなたも変態。 いいね? _ _ _ _ _ __ | | | | ___ _ __ | |_ __ _(_) | '_ \| |_| |/ _ \ '_ \| __/ _` | | | | | | _ | __/ | | | || (_| | | |_| |_|_| |_|\___|_| |_|\__\__,_|_| ''')
def banner(): logger.info(u'''nHentai ver %s: あなたも変態。 いいね? _ _ _ _ _ __ | | | | ___ _ __ | |_ __ _(_) | '_ \| |_| |/ _ \ '_ \| __/ _` | | | | | | _ | __/ | | | || (_| | | |_| |_|_| |_|\___|_| |_|\__\__,_|_| ''' % __version__)
def banner(): logger.info('''nHentai: あなたも変態。 いいね? _ _ _ _ _ __ | | | | ___ _ __ | |_ __ _(_) | '_ \| |_| |/ _ \ '_ \| __/ _` | | | | | | _ | __/ | | | || (_| | | |_| |_|_| |_|\___|_| |_|\__\__,_|_| ''')
def login_parser(username, password): s = requests.Session() s.proxies = constant.PROXY s.verify = False s.headers.update({'Referer': constant.LOGIN_URL}) s.get(constant.LOGIN_URL) content = s.get(constant.LOGIN_URL).content html = BeautifulSoup(content, 'html.parser') csrf_token_elem = html.find('input', attrs={'name': 'csrfmiddlewaretoken'}) if not csrf_token_elem: raise Exception('Cannot find csrf token to login') csrf_token = csrf_token_elem.attrs['value'] login_dict = { 'csrfmiddlewaretoken': csrf_token, 'username_or_email': username, 'password': password, } resp = s.post(constant.LOGIN_URL, data=login_dict) if 'Invalid username (or email) or password' in resp.text: logger.error('Login failed, please check your username and password') exit(1) html = BeautifulSoup(s.get(constant.FAV_URL).content, 'html.parser') count = html.find('span', attrs={'class': 'count'}) if not count: logger.error('Cannot get count of your favorites, maybe login failed.') count = int(count.text.strip('(').strip(')')) pages = count / 25 pages += 1 if count % (25 * pages) else 0 logger.info('Your have %d favorites in %d pages.' % (count, pages)) if os.getenv('DEBUG'): pages = 1 ret = [] doujinshi_id = re.compile('data-id="([\d]+)"') def _callback(request, result): ret.append(result) thread_pool = threadpool.ThreadPool(5) for page in range(1, pages+1): try: logger.info('Getting doujinshi id of page %d' % page) resp = s.get(constant.FAV_URL + '?page=%d' % page).content ids = doujinshi_id.findall(resp) requests_ = threadpool.makeRequests(doujinshi_parser, ids, _callback) [thread_pool.putRequest(req) for req in requests_] thread_pool.wait() except Exception as e: logger.error('Error: %s, continue', str(e)) return ret
def download(self): logger.info('Start download doujinshi: %s' % self.name) if self.downloader: download_queue = [] for i in range(1, self.pages + 1): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext)) self.downloader.download(download_queue, format_filename('%s-%s' % (self.id, self.name[:200]))) else: logger.critical('Downloader has not be loaded')
def download(self): logger.info('Start download doujinshi: %s' % self.name) if self.downloader: download_queue = [] for i in range(1, self.pages + 1): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext)) self.downloader.download(download_queue, self.id) else: logger.critical('Downloader has not be loaded')
def search_parser(keyword, sorting, page, is_page_all=False): # keyword = '+'.join([i.strip().replace(' ', '-').lower() for i in keyword.split(',')]) result = [] response = None if not page: page = [1] if is_page_all: url = request('get', url=constant.SEARCH_URL, params={ 'query': keyword }).url init_response = request('get', url.replace('%2B', '+')).json() page = range(1, init_response['num_pages'] + 1) total = '/{0}'.format(page[-1]) if is_page_all else '' not_exists_persist = False for p in page: i = 0 logger.info( 'Searching doujinshis using keywords "{0}" on page {1}{2}'.format( keyword, p, total)) while i < 3: try: url = request('get', url=constant.SEARCH_URL, params={ 'query': keyword, 'page': p, 'sort': sorting }).url response = request('get', url.replace('%2B', '+')).json() except Exception as e: logger.critical(str(e)) response = None break if response is None or 'result' not in response: logger.warning('No result in response in page {}'.format(p)) if not_exists_persist is True: break continue for row in response['result']: title = row['title']['english'] title = title[:85] + '..' if len(title) > 85 else title result.append({'id': row['id'], 'title': title}) not_exists_persist = False if not result: logger.warning('No results for keywords {}'.format(keyword)) return result
def main(): banner() logger.info('Using mirror: {0}'.format(BASE_URL)) options = cmd_parser() doujinshi_ids = [] doujinshi_list = [] if options.favorites: if not options.is_download: logger.warning('You do not specify --download option') doujinshi_ids = favorites_parser() elif options.tag: doujinshis = tag_parser(options.tag, max_page=options.max_page) print_doujinshi(doujinshis) if options.is_download and doujinshis: doujinshi_ids = map(lambda d: d['id'], doujinshis) elif options.keyword: doujinshis = search_parser(options.keyword, options.page) print_doujinshi(doujinshis) if options.is_download: doujinshi_ids = map(lambda d: d['id'], doujinshis) elif not doujinshi_ids: doujinshi_ids = options.id if doujinshi_ids: for id_ in doujinshi_ids: if options.delay: time.sleep(options.delay) doujinshi_info = doujinshi_parser(id_) doujinshi_list.append(Doujinshi(name_format=options.name_format, path=options.output_dir, **doujinshi_info)) if not options.is_show: downloader = Downloader(path=options.output_dir, thread=options.threads, timeout=options.timeout, delay=options.delay) for doujinshi in doujinshi_list: doujinshi.downloader = downloader doujinshi.download() if not options.is_nohtml and not options.is_cbz: generate_html(options.output_dir, doujinshi) elif options.is_cbz: generate_cbz(options.output_dir, doujinshi, options.rm_origin_dir) if not platform.system() == 'Windows': logger.log(15, '🍻 All done.') else: logger.log(15, 'All done.') else: [doujinshi.show() for doujinshi in doujinshi_list]
def show(self): table = [ ["Doujinshi", self.name], ["Subtitle", self.info.subtitle], ["Characters", self.info.character], ["Authors", self.info.artist], ["Language", self.info.language], ["Tags", ', '.join(self.info.tag.keys())], ["URL", self.url], ["Pages", self.pages], ] logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(table)))
def show(self): table = [ ["Doujinshi", self.name], ["Subtitle", self.info.subtitle], ["Characters", self.info.characters], ["Authors", self.info.artists], ["Language", self.info.language], ["Tags", self.info.tags], ["URL", self.url], ["Pages", self.pages], ] logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(table)))
def generate_metadatafile(output_dir, table, doujinshi_obj=None): logger.info("Writing Metadata Info") if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) else: doujinshi_dir = '.' logger.info(doujinshi_dir) f = open(os.path.join(doujinshi_dir, 'info.txt'), "w", encoding="utf-8") fields = [ "TITLE", "ORIGINAL TITLE", "AUTHOR", "ARTIST", "CIRCLE", "SCANLATOR", "TRANSLATOR", "PUBLISHER", "DESCRIPTION", "STATUS", "CHAPTERS", "PAGES", "TAGS", "TYPE", "LANGUAGE", "RELEASED", "READING DIRECTION", "CHARACTERS", "SERIES", "PARODY", "URL" ] for i in range(21): f.write("%s: " % fields[i]) if (i == 19): f.write("%s" % table[0][1]) if (i == 0): f.write("%s" % table[1][1]) if (i == 1): f.write("%s" % table[2][1]) if (i == 17): f.write("%s" % table[3][1]) if (i == 2): f.write("%s" % table[4][1]) if (i == 14): f.write("%s" % table[5][1]) if (i == 12): f.write("%s" % table[6][1]) if (i == 20): f.write("%s" % table[7][1]) if (i == 11): f.write("%s" % table[8][1]) f.write("\n") f.close()
def download(self): logger.info('Starting to download doujinshi: %s' % self.name) if self.downloader: download_queue = [] if len(self.ext) != self.pages: logger.warning('Page count and ext count do not equal') for i in range(1, min(self.pages, len(self.ext)) + 1): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext[i - 1])) self.downloader.download(download_queue, self.filename) else: logger.critical('Downloader has not been loaded')
def tag_parser(tag_id, max_page=1): logger.info('Searching for doujinshi with tag id {0}'.format(tag_id)) result = [] i = 0 while i < 5: try: response = request('get', url=constant.TAG_API_URL, params={ 'sort': 'popular', 'tag_id': tag_id }).json() except Exception as e: i += 1 if not i < 5: logger.critical(str(e)) exit(1) continue break page = max_page if max_page <= response['num_pages'] else int( response['num_pages']) for i in range(1, page + 1): logger.info('Getting page {} ...'.format(i)) if page != 1: i = 0 while i < 5: try: response = request('get', url=constant.TAG_API_URL, params={ 'sort': 'popular', 'tag_id': tag_id }).json() except Exception as e: i += 1 if not i < 5: logger.critical(str(e)) exit(1) continue break for row in response['result']: title = row['title']['english'] title = title[:85] + '..' if len(title) > 85 else title result.append({'id': row['id'], 'title': title}) if not result: logger.warn('No results for tag id {}'.format(tag_id)) return result
def tag_parser(tag_id): logger.info('Get doujinshi of tag id: {0}'.format(tag_id)) result = [] response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json() for row in response['result']: title = row['title']['english'] title = title[:85] + '..' if len(title) > 85 else title result.append({'id': row['id'], 'title': title}) if not result: logger.warn('Not found anything of tag id {}'.format(tag_id)) return result
def check_cookie(): response = request('get', constant.BASE_URL) if response.status_code == 503 and 'cf-browser-verification' in response.text: logger.error( 'Blocked by Cloudflare captcha, please set your cookie and useragent' ) exit(-1) username = re.findall('"/users/\d+/(.*?)"', response.text) if not username: logger.warning( 'Cannot get your username, please check your cookie or use `nhentai --cookie` to set your cookie' ) else: logger.info('Login successfully! Your username: {}'.format( username[0]))
def login_parser(): html = BeautifulSoup( request('get', constant.FAV_URL).content, 'html.parser') count = html.find('span', attrs={'class': 'count'}) if not count: logger.error( "Can't get your number of favorited doujins. Did the login failed?" ) return count = int(count.text.strip('(').strip(')').replace(',', '')) if count == 0: logger.warning('No favorites found') return [] pages = int(count / 25) if pages: pages += 1 if count % (25 * pages) else 0 else: pages = 1 logger.info('You have %d favorites in %d pages.' % (count, pages)) if os.getenv('DEBUG'): pages = 1 ret = [] doujinshi_id = re.compile('data-id="([\d]+)"') def _callback(request, result): ret.append(result) thread_pool = threadpool.ThreadPool(5) for page in range(1, pages + 1): try: logger.info('Getting doujinshi ids of page %d' % page) resp = request('get', constant.FAV_URL + '?page=%d' % page).text ids = doujinshi_id.findall(resp) requests_ = threadpool.makeRequests(doujinshi_parser, ids, _callback) [thread_pool.putRequest(req) for req in requests_] thread_pool.wait() except Exception as e: logger.error('Error: %s, continue', str(e)) return ret
def login_parser(): html = BeautifulSoup(request('get', constant.FAV_URL).content, 'html.parser') count = html.find('span', attrs={'class': 'count'}) if not count: logger.error("Can't get your number of favorited doujins. Did the login failed?") return count = int(count.text.strip('(').strip(')').replace(',', '')) if count == 0: logger.warning('No favorites found') return [] pages = int(count / 25) if pages: pages += 1 if count % (25 * pages) else 0 else: pages = 1 logger.info('You have %d favorites in %d pages.' % (count, pages)) if os.getenv('DEBUG'): pages = 1 ret = [] doujinshi_id = re.compile('data-id="([\d]+)"') def _callback(request, result): ret.append(result) thread_pool = threadpool.ThreadPool(5) for page in range(1, pages + 1): try: logger.info('Getting doujinshi ids of page %d' % page) resp = request('get', constant.FAV_URL + '?page=%d' % page).text ids = doujinshi_id.findall(resp) requests_ = threadpool.makeRequests(doujinshi_parser, ids, _callback) [thread_pool.putRequest(req) for req in requests_] thread_pool.wait() except Exception as e: logger.error('Error: %s, continue', str(e)) return ret
def favorites_parser(page=None): result = [] html = BeautifulSoup( request('get', constant.FAV_URL).content, 'html.parser') count = html.find('span', attrs={'class': 'count'}) if not count: logger.error( "Can't get your number of favorited doujins. Did the login failed?" ) return [] count = int(count.text.strip('(').strip(')').replace(',', '')) if count == 0: logger.warning('No favorites found') return [] pages = int(count / 25) if page: page_range_list = page else: if pages: pages += 1 if count % (25 * pages) else 0 else: pages = 1 logger.info('You have %d favorites in %d pages.' % (count, pages)) if os.getenv('DEBUG'): pages = 1 page_range_list = range(1, pages + 1) for page in page_range_list: try: logger.info('Getting doujinshi ids of page %d' % page) resp = request('get', constant.FAV_URL + '?page=%d' % page).content result.extend(_get_title_and_id(resp)) except Exception as e: logger.error('Error: %s, continue', str(e)) return result
def download(self): logger.info('Starting to download doujinshi: %s' % self.name) if self.downloader: download_queue = [] if len(self.ext) != self.pages: logger.warning('Page count and ext count do not equal') for i in range(1, min(self.pages, len(self.ext)) + 1): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i, self.ext[i-1])) self.downloader.download(download_queue, self.filename) ''' for i in range(len(self.ext)): download_queue.append('%s/%d/%d.%s' % (IMAGE_URL, int(self.img_id), i+1, EXT_MAP[self.ext[i]])) ''' else: logger.critical('Downloader has not been loaded')
def generate_cbz(output_dir='.', doujinshi_obj=None, rm_origin_dir=False): if doujinshi_obj is not None: doujinshi_dir = os.path.join(output_dir, doujinshi_obj.filename) cbz_filename = os.path.join(os.path.join(doujinshi_dir, '..'), '%s.cbz' % doujinshi_obj.id) else: cbz_filename = './doujinshi.cbz' doujinshi_dir = '.' file_list = os.listdir(doujinshi_dir) file_list.sort() logger.info('Writing CBZ file to path: {}'.format(cbz_filename)) with zipfile.ZipFile(cbz_filename, 'w') as cbz_pf: for image in file_list: image_path = os.path.join(doujinshi_dir, image) cbz_pf.write(image_path, image) if rm_origin_dir: shutil.rmtree(doujinshi_dir, ignore_errors=True) logger.log(15, 'Comic Book CBZ file has been write to \'{0}\''.format(doujinshi_dir))
def __api_suspended_tag_parser(tag_id, max_page=1): logger.info('Searching for doujinshi with tag id {0}'.format(tag_id)) result = [] response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json() page = max_page if max_page <= response['num_pages'] else int(response['num_pages']) for i in range(1, page + 1): logger.info('Getting page {} ...'.format(i)) if page != 1: response = request('get', url=constant.TAG_API_URL, params={'sort': 'popular', 'tag_id': tag_id}).json() for row in response['result']: title = row['title']['english'] title = title[:85] + '..' if len(title) > 85 else title result.append({'id': row['id'], 'title': title}) if not result: logger.warn('No results for tag id {}'.format(tag_id)) return result
def main(): banner() logger.info('Using mirror: {0}'.format(BASE_URL)) options = cmd_parser() doujinshi_ids = [] doujinshi_list = [] if options.keyword: doujinshis = search_parser(options.keyword, options.page) print_doujinshi(doujinshis) if options.is_download: doujinshi_ids = map(lambda d: d['id'], doujinshis) else: doujinshi_ids = options.id if doujinshi_ids: for id in doujinshi_ids: doujinshi_info = doujinshi_parser(id) doujinshi_list.append(Doujinshi(**doujinshi_info)) else: exit(0) if not options.is_show: downloader = Downloader(path=options.output_dir, thread=options.threads, timeout=options.timeout) for doujinshi in doujinshi_list: doujinshi.downloader = downloader doujinshi.download() generate_html(options.output_dir, doujinshi) if not platform.system() == 'Windows': logger.log(15, '🍺 All done.') else: logger.log(15, 'All done.') else: [doujinshi.show() for doujinshi in doujinshi_list]
def main(): banner() logger.info('Using mirror: {0}'.format(BASE_URL)) options = cmd_parser() doujinshi_ids = [] doujinshi_list = [] if options.keyword: doujinshis = search_parser(options.keyword, options.page) print_doujinshi(doujinshis) if options.is_download: doujinshi_ids = map(lambda d: d['id'], doujinshis) else: doujinshi_ids = options.id if doujinshi_ids: for id in doujinshi_ids: doujinshi_info = doujinshi_parser(id) doujinshi_list.append(Doujinshi(**doujinshi_info)) else: exit(0) if not options.is_show: downloader = Downloader(path=options.output_dir, thread=options.threads, timeout=options.timeout) for doujinshi in doujinshi_list: doujinshi.downloader = downloader doujinshi.download() if not platform.system() == 'Windows': logger.log(15, '🍺 All done.') else: logger.log(15, 'All done.') else: [doujinshi.show() for doujinshi in doujinshi_list]
def show(self): logger.info(u'Print doujinshi information of {0}\n{1}'.format(self.id, tabulate(self.table)))