def map_external_gallery_data_to_internal( gallery_data: DataDict) -> GalleryData: internal_gallery_data = GalleryData( gallery_data['gid'], token=gallery_data['token'], archiver_key=gallery_data['archiver_key'], title=unescape(gallery_data['title']), title_jpn=unescape(gallery_data['title_jpn']), thumbnail_url=gallery_data['thumb'], category=gallery_data['category'], provider=constants.provider_name, uploader=gallery_data['uploader'], posted=datetime.fromtimestamp(int(gallery_data['posted']), timezone.utc), filecount=gallery_data['filecount'], filesize=gallery_data['filesize'], expunged=gallery_data['expunged'], rating=gallery_data['rating'], tags=translate_tag_list(gallery_data['tags']), ) m = re.search(constants.default_fjord_tags, ",".join(internal_gallery_data.tags)) if m: internal_gallery_data.fjord = True if internal_gallery_data.thumbnail_url and constants.ex_thumb_url in internal_gallery_data.thumbnail_url: internal_gallery_data.thumbnail_url = internal_gallery_data.thumbnail_url.replace( constants.ex_thumb_url, constants.ge_thumb_url) return internal_gallery_data
def get_values_from_gallery_link(self, link: str) -> Optional[GalleryData]: request_dict = construct_request_dict(self.settings, self.own_settings) response = request_with_retries( link, request_dict, post=False, ) if not response: return None response.encoding = 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') if soup: title_jpn_match = soup.find("div", id=re.compile("info")).h2 gallery_id_match = re.search( r'{}(\d+)'.format(constants.gallery_container_url), link) if not gallery_id_match: return None gallery_id = 'nh-' + gallery_id_match.group(1) gallery = GalleryData(gallery_id, self.name) gallery.title = soup.h1.get_text() gallery.title_jpn = title_jpn_match.get_text( ) if title_jpn_match else '' gallery_filecount_match = re.search(r'<div>(\d+) page(s*)</div>', response.text) if gallery_filecount_match: gallery.filecount = int(gallery_filecount_match.group(1)) else: gallery.filecount = 0 gallery.tags = [] gallery.link = link gallery.posted = dateutil.parser.parse( soup.find("time")['datetime']) for tag_container in soup.find_all("a", {"class": "tag"}): tag_name = [text for text in tag_container.stripped_strings][0] tag_name = tag_name.split(" | ")[0] tag_scope = tag_container.parent.parent.get_text() tag_ext = tag_container.parent.get_text() tag_scope = tag_scope.replace(tag_ext, "").replace( "\t", "").replace("\n", "").replace(":", "").lower() if tag_scope == 'tags': gallery.tags.append(translate_tag(tag_name)) elif tag_scope == 'categories': gallery.category = tag_name.capitalize() else: gallery.tags.append( translate_tag(tag_scope + ":" + tag_name)) else: return None return gallery
def test_nhentai_parser(self): """Test Nhentai gallery page parser""" settings = Settings(load_from_disk=True) gallery_link = 'https://nhentai.net/g/198482/' parser = NhentaiParser(settings) data = parser.fetch_gallery_data(gallery_link) expected_data = GalleryData( 'nh-198482', 'nhentai', title= '(C90) [MeltdoWN COmet (Yukiu Con)] C90 Omakebon! (Pokémon GO) [English] [ATF]', title_jpn='(C90) [MeltdoWN COmet (雪雨こん)] C90 おまけ本! (ポケモンGO) [英訳]', filecount=9, link='https://nhentai.net/g/198482/', posted=dateutil.parser.parse('2017-06-19T10:33:19.022360+00:00'), category='Doujinshi', tags=[ 'parody:pokemon', 'lolicon', 'sole_female', 'sole_male', 'b*****b', 'artist:yukiu_con', 'group:meltdown_comet', 'language:translated', 'language:english', ]) self.assertEqual(data, expected_data)
def crawl_urls(self, urls: List[str], wanted_filters=None, wanted_only: bool = False) -> None: unique_urls = set() gallery_data_list = [] gallery_wanted_lists = defaultdict(list) if not self.downloaders: self.logger.warning('No downloaders enabled, returning.') return for url in urls: if not any(word in url for word in self.accepted_urls): self.logger.warning("Invalid URL, skipping: {}".format(url)) continue url = url.replace(constants.base_url, constants.old_base_url) unique_urls.add(url) for gallery_url in unique_urls: gallery_data = GalleryData(gallery_url, link=gallery_url) gallery_data_list.append(gallery_data) self.pass_gallery_data_to_downloaders(gallery_data_list, gallery_wanted_lists)
def compare_by_title(self, gallery_title: str) -> bool: api_url = urljoin(self.own_settings.metadata_url, constants.api_path) logger.info("Querying URL: {}".format(api_url)) request_dict = construct_request_dict(self.settings, self.own_settings) request_dict['params'] = {'match': True, 'title': gallery_title} response = request_with_retries( api_url, request_dict, post=False, retries=3 ) if not response: logger.info("Got no response from server") return False response_data = response.json() matches_links = set() if 'error' in response_data: logger.info("Got error from server: {}".format(response_data['error'])) return False for gallery in response_data: if 'link' in gallery: matches_links.add(gallery['link']) if 'gallery_container' in gallery and gallery['gallery_container']: if self.settings.gallery_model: gallery_container = self.settings.gallery_model.objects.filter( gid=gallery['gallery_container'], provider=gallery['provider'] ) first_gallery_container = gallery_container.first() if first_gallery_container: gallery['gallery_container_gid'] = first_gallery_container.gid if 'magazine' in gallery and gallery['magazine']: if self.settings.gallery_model: magazine = self.settings.gallery_model.objects.filter( gid=gallery['magazine'], provider=gallery['provider'] ) first_magazine = magazine.first() if first_magazine: gallery['magazine_gid'] = first_magazine.gid if 'posted' in gallery: if gallery['posted'] != 0: gallery['posted'] = datetime.fromtimestamp(int(gallery['posted']), timezone.utc) else: gallery['posted'] = None self.values_array.append(GalleryData(**gallery)) self.gallery_links = list(matches_links) if len(self.gallery_links) > 0: self.found_by = self.name return True else: return False
def crawl_urls(self, urls: list[str], wanted_filters=None, wanted_only: bool = False) -> None: unique_urls = set() gallery_data_list = [] gallery_wanted_lists: dict[str, list['WantedGallery']] = defaultdict(list) if not self.downloaders: logger.warning('No downloaders enabled, returning.') return for url in urls: unique_urls.add(url) for gallery_url in unique_urls: gallery_data = GalleryData(gallery_url, self.name, link=gallery_url) gallery_data_list.append(gallery_data) self.pass_gallery_data_to_downloaders(gallery_data_list, gallery_wanted_lists)
def crawl_urls(self, urls: list[str], wanted_filters=None, wanted_only: bool = False) -> None: unique_urls = set() gallery_data_list = [] gallery_wanted_lists: dict[str, list['WantedGallery']] = defaultdict(list) if not self.downloaders: logger.warning('No downloaders enabled, returning.') return for url in urls: if not any(word in url for word in self.accepted_urls): logger.warning("Invalid URL, skipping: {}".format(url)) continue if constants.torrent_download_path in url: utilities.view_link_from_download_link(url) unique_urls.add(url) for gallery in unique_urls: gid = self.id_from_url(gallery) if not gid: continue discard_approved, discard_message = self.discard_gallery_by_internal_checks( gallery_id=gid, link=gallery ) if discard_approved: if not self.settings.silent_processing: logger.info(discard_message) continue gallery_data = GalleryData(gid, self.name, link=gallery) gallery_data_list.append(gallery_data) if not gallery_data_list: return self.pass_gallery_data_to_downloaders(gallery_data_list, gallery_wanted_lists)
def test_nexus_parser(self): """Test Nexus gallery page parser""" settings = Settings(load_from_disk=True) gallery_link = 'https://hentainexus.com/view/5665' parser = NexusParser(settings) data = parser.fetch_gallery_data(gallery_link) expected_data = GalleryData( '5665', 'nexus', link=gallery_link, archiver_key='https://hentainexus.com/zip/5665', title='Sase-san is Very Popular', thumbnail_url= 'https://static.hentainexus.com/content/5665/cover.jpg', filecount=16, filesize=0, expunged=False, posted=None, category='Manga', tags=[ 'artist:wantan_meo', 'language:english', 'magazine:comic_kairakuten_2019-04', 'parody:original_work', 'publisher:fakku', 'creampie', 'fangs', 'hairy', 'hentai', 'office_lady', 'oppai', 'uncensored', 'vanilla', ], comment='Let\'s chug \'em down! ♪', ) self.assertEqual(data, expected_data)
def get_values_from_gallery_link(self, link: str) -> Optional[GalleryData]: request_dict = construct_request_dict(self.settings, self.own_settings) response = request_with_retries( link, request_dict, post=False, ) if not response: return None response.encoding = 'utf-8' match_string = re.compile(constants.main_page + '/(.+)/$') tags = [] soup = BeautifulSoup(response.text, 'html.parser') content_container = soup.find("div", class_="content") if not content_container: return None artists_container = content_container.find_all( "a", href=re.compile(constants.main_page + '/artist/.*/$')) for artist in artists_container: tags.append("artist:{}".format(artist.get_text())) tags_container = content_container.find_all( "a", href=re.compile(constants.main_page + '/tag/.*/$')) for tag in tags_container: tags.append(tag.get_text()) # thumbnail_small_container = content_container.find("img") # if thumbnail_small_container: # thumbnail_url = thumbnail_small_container.get('src') thumbnail_url = soup.find("meta", property="og:image") match_result = match_string.match(soup.find("meta", property="og:link")) if not match_result: return None gallery = GalleryData( match_result.group(1), self.name, link=link, title=soup.find("meta", property="og:title"), comment='', thumbnail_url=thumbnail_url, category='Manga', uploader='', posted=None, filecount=0, filesize=0, expunged=False, rating='', tags=translate_tag_list(tags), content=content_container.encode_contents(), ) return gallery
def crawl_feed(self, feed_url: str = '') -> list[GalleryData]: if not feed_url: feed_url = constants.rss_url request_dict = construct_request_dict(self.settings, self.own_settings) response = request_with_retries( feed_url, request_dict, post=False, ) if not response: logger.error("Got no response from feed URL: {}".format(feed_url)) return [] response.encoding = 'utf-8' feed = feedparser.parse(response.text) galleries = [] match_string = re.compile(constants.main_page + '/(.+)/$') skip_tags = ['Uncategorized'] logger.info( "Provided RSS URL for provider ({}), adding {} found links".format( self.name, len(feed['items']))) for item in feed['items']: tags = [x.term for x in item['tags'] if x.term not in skip_tags] thumbnail_url = '' for content in item['content']: soup = BeautifulSoup(content.value, 'html.parser') artists_container = soup.find_all( "a", href=re.compile(constants.main_page + '/artist/.*/$')) for artist in artists_container: tags.append("artist:{}".format(artist.get_text())) thumbnail_small_container = soup.find("img") if thumbnail_small_container: thumbnail_url = thumbnail_small_container.get('src') match_result = match_string.match(item['link']) if not match_result: continue gallery = GalleryData(match_result.group(1), self.name, title=item['title'], comment=item['description'], thumbnail_url=thumbnail_url, category='Manga', uploader=item['author'], posted=datetime.strptime( item['published'], "%a, %d %b %Y %H:%M:%S %z"), filecount=0, filesize=0, expunged=False, rating='', tags=translate_tag_list(tags), content=item['content'][0].value, link=item['link']) # Must check here since this method is called after the main check in crawl_urls if self.general_utils.discard_by_tag_list(gallery.tags): continue if not gallery.link: continue discard_approved, discard_message = self.discard_gallery_by_internal_checks( gallery.gid, link=gallery.link) if discard_approved: if not self.settings.silent_processing: logger.info(discard_message) continue galleries.append(gallery) return galleries
def work_gallery_data(self, gallery: GalleryData, gallery_wanted_lists) -> None: if not gallery.token: return m = re.search(constants.default_fjord_tags, ",".join(gallery.tags)) if m: gallery.root = constants.ex_page gallery.link = link_from_gid_token_fjord(gallery.gid, gallery.token, True) else: gallery.root = constants.ge_page gallery.link = link_from_gid_token_fjord(gallery.gid, gallery.token, False) self.logger.info("Title: {}. Link: {}".format(gallery.title, gallery.link)) gallery_is_hidden = False gallery.hidden = False if self.own_settings.crawl_gallery_page: gallery_page_text = requests.get( gallery.link, cookies=self.own_settings.cookies, headers=self.settings.requests_headers, timeout=self.settings.timeout_timer).text if 'Gallery Not Available' in gallery_page_text: if gallery.root == constants.ex_page: self.logger.warning( 'EX Gallery not available, probably taken down/hidden') gallery_is_hidden = True gallery.hidden = True elif gallery.root == constants.ge_page: time.sleep(self.settings.wait_timer) gallery.root = constants.ex_page gallery.link = link_from_gid_token_fjord( gallery.gid, gallery.token, True) retry_gallery_page_text = requests.get( gallery.link, cookies=self.own_settings.cookies, headers=self.settings.requests_headers, timeout=self.settings.timeout_timer).text if 'Gallery Not Available' in retry_gallery_page_text: self.logger.warning( 'Tried EX gallery instead of E-H for ' 'arbitrary hidden galleries, also not ' 'available, probably taken down/hidden') gallery_is_hidden = True gallery.hidden = True else: self.logger.info( "Changed from E-H to EX, was a arbitrary hidden gallery" ) gallery_page_text = retry_gallery_page_text time.sleep(self.settings.wait_timer) new_gallery = self.fetch_gallery_data(gallery.link) if new_gallery and new_gallery.token: gallery = new_gallery gallery.root = constants.ex_page gallery.link = link_from_gid_token_fjord( new_gallery.gid, new_gallery.token, True) time.sleep(self.settings.wait_timer) if not gallery_is_hidden: gallery_parser = GalleryHTMLParser() gallery_parser.feed(gallery_page_text) # What this does is log a parent gallery if we are downloading a newer one. # Maybe this should be an option, because we are deleting files from disk. if gallery_parser.found_parent_gallery and gallery_parser.parent_gallery: parent_gid, parent_token = get_gid_token_from_link( gallery_parser.parent_gallery) archives = Archive.objects.filter( gallery__gid__exact=parent_gid, gallery__token__exact=parent_token) for archive in archives: self.logger.warning( "Gallery: {} has a parent gallery {}, " "which is matched with archive: {}. You might want to delete them." .format(gallery.link, archive.gallery.get_absolute_url(), archive.get_absolute_url())) if (gallery_parser.found_non_final_gallery == 2 and self.own_settings.get_newer_gallery and gallery_parser.non_final_gallery): self.logger.info( "Found non final gallery, next is: {}".format( gallery_parser.non_final_gallery)) (exists_next, new_gallery) = self.get_final_gallery_from_link( gallery_parser.non_final_gallery) if exists_next == 1 and new_gallery and new_gallery.link: gallery.dl_type = 'skipped:non_final' Gallery.objects.update_or_create_from_values(gallery) discard_approved, discard_message = self.discard_gallery_by_internal_checks( gallery_id=new_gallery.gid, link=new_gallery.link) if discard_approved: if not self.settings.silent_processing: self.logger.info(discard_message) return gallery = new_gallery self.logger.info("Final gallery is: {}".format( new_gallery.link)) elif exists_next == 0: self.logger.info( "Last in sequence: {} is not available, using current one" .format(gallery.link)) elif exists_next == 2: self.logger.info( "Last in sequence: {} was discarded by global tags, " "skipping this gallery altogether".format( gallery.link)) gallery.dl_type = 'skipped:final_replaced' Gallery.objects.update_or_create_from_values(gallery) return retry_attempt = True while retry_attempt: for cnt, downloader in enumerate(self.downloaders): if not gallery_is_hidden or not downloader[0].skip_if_hidden: downloader[0].init_download(copy.deepcopy(gallery)) else: downloader[0].return_code = 0 if downloader[0].return_code == 1: for wanted_gallery in gallery_wanted_lists[gallery.gid]: FoundGallery.objects.get_or_create( wanted_gallery=wanted_gallery, gallery=downloader[0].gallery_db_entry) if wanted_gallery.add_as_hidden and downloader[ 0].gallery_db_entry: downloader[0].gallery_db_entry.hidden = True downloader[0].gallery_db_entry.save() if downloader[ 0].archive_db_entry and wanted_gallery.reason: downloader[ 0].archive_db_entry.reason = wanted_gallery.reason downloader[0].archive_db_entry.simple_save() if len(gallery_wanted_lists[gallery.gid]) > 0: wanted_gallery_found.send( sender=self.settings.gallery_model, gallery=downloader[0].gallery_db_entry, wanted_gallery_list=gallery_wanted_lists[ gallery.gid]) self.last_used_downloader = str(downloader[0]) if downloader[0].gallery_db_entry: if downloader[0].archive_db_entry: self.logger.info( "Download completed successfully, using {}. Archive link: {}. Gallery link: {}" .format( downloader[0], downloader[0]. archive_db_entry.get_absolute_url(), downloader[0].gallery_db_entry. get_absolute_url())) else: self.logger.info( "Download completed successfully (gallery only), using {}. Gallery link: {}" .format( downloader[0], downloader[0]. gallery_db_entry.get_absolute_url())) return if (downloader[0].return_code == 0 and (cnt + 1) == len(self.downloaders)): if gallery.root == constants.ge_page and not gallery_is_hidden and gallery.token: gallery.root = constants.ex_page gallery.link = link_from_gid_token_fjord( gallery.gid, gallery.token, True) # fetch archiver key again when retrying. new_gallery_data = self.fetch_gallery_data( gallery.link) if new_gallery_data: gallery.archiver_key = new_gallery_data.archiver_key self.logger.info( "Retrying with fjord link, might be hidden.") break else: self.logger.error("Could not fetch fjord link.") else: self.logger.error( "Finished retrying using fjord link.") downloader[0].original_gallery = gallery downloader[0].original_gallery.hidden = True downloader[0].original_gallery.dl_type = 'failed' downloader[0].update_gallery_db() if downloader[0].gallery_db_entry: self.last_used_downloader = 'none' self.logger.warning( "Download completed unsuccessfully, set as failed. Gallery link: {}" .format(downloader[0].gallery_db_entry. get_absolute_url())) for wanted_gallery in gallery_wanted_lists[ gallery.gid]: FoundGallery.objects.get_or_create( wanted_gallery=wanted_gallery, gallery=downloader[0].gallery_db_entry) retry_attempt = False
def get_values_from_gallery_link(self, link: str) -> Optional[GalleryData]: request_dict = construct_request_dict(self.settings, self.own_settings) response = request_with_retries( link, request_dict, post=False, ) if not response: return None response.encoding = 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') gallery_container_head = soup.find("head") if gallery_container_head: gid_container = gallery_container_head.find("meta", property="og:url") if gid_container: url_parts = gid_container['content'].split('/') gid = url_parts[-1] gallery = GalleryData(gid, self.name) gallery.link = link gallery.tags = [] gallery.category = 'Doujinshi' title_container = gallery_container_head.find( "meta", property="og:title") image_url_container = gallery_container_head.find( "meta", property="og:image") tags_containers = gallery_container_head.find_all( "meta", property="book:tag") page_count_container = gallery_container_head.find( "meta", property="books:page_count") description_container = gallery_container_head.find( "meta", property="og:description") author_container = soup.find("span", itemprop="author") section_container = soup.find( "section", class_="showcase_comic_single_description") gallery_container_titles = soup.find( "hgroup", class_='showcase_comic_single_main_titles') if title_container: gallery.title = title_container['content'] if gallery_container_titles: title_jpn_container = gallery_container_titles.find( "span", lang='ja') if title_jpn_container: gallery.title_jpn = title_jpn_container.get_text() if image_url_container: gallery.thumbnail_url = image_url_container['content'] if description_container: gallery.comment = description_container['content'] if page_count_container: gallery.filecount = int(page_count_container['content']) if author_container: group_name = author_container.find("meta", itemprop="name") if group_name: gallery.tags.append( translate_tag("group:" + group_name['content'])) if section_container: time_container = section_container.find( "time", itemprop="datePublished") if time_container: gallery.posted = datetime.fromisoformat( time_container['datetime'] + '+00:00') p_containers = section_container.find_all("p") for ps in p_containers: p_text = ps.get_text() if 'Source:' in p_text: parody_name = p_text.replace('Source: ', '').rstrip() gallery.tags.append( translate_tag("parody:" + parody_name)) for tag_container in tags_containers: tag = translate_tag(tag_container['content']) gallery.tags.append(tag) gallery.tags.append(translate_tag("language:english")) else: return None else: return None return gallery
def test_fakku_parser(self): """Test FAKKU gallery page parser""" settings = Settings(load_from_disk=True) gallery_link = 'https://www.fakku.net/hentai/im-a-piece-of-junk-sexaroid-english' parser = FakkuParser(settings) data = parser.fetch_gallery_data(gallery_link) expected_data = GalleryData( 'hentai/im-a-piece-of-junk-sexaroid-english', 'fakku', link=gallery_link, title='I\'m a Piece of Junk Sexaroid', thumbnail_url= 'https://t.fakku.net/images/manga/i/im-a-piece-of-junk-sexaroid-english/thumbs/002.thumb.jpg', filecount=16, category='Manga', tags=[ 'artist:wakame-san', 'magazine:comic_kairakuten_beast_2017-05', 'publisher:fakku', 'language:english', 'tsundere', 'femdom', 'vanilla', 'b*****b', 'oppai', 'hentai', 'creampie', 'uncensored', 'x-ray', 'subscription', ], comment='Plump slacker sex robot ❤', ) self.assertEqual(data, expected_data) gallery_link = 'https://www.fakku.net/hentai/tsf-story-append-20-english_1497401155' parser = FakkuParser(settings) data = parser.fetch_gallery_data(gallery_link) expected_data = GalleryData( 'hentai/tsf-story-append-20-english_1497401155', 'fakku', link=gallery_link, title='TSF Story Append 2.0', filecount=82, category='Doujinshi', tags=[ 'artist:oda_non', 'artist:yasui_riosuke', 'artist:meme50', 'artist:kojima_saya', 'artist:butcha-u', 'artist:mizuryu_kei', 'artist:kurenai_yuuji', 'artist:soine', 'artist:asanagi', 'artist:yumeno_tanuki', 'artist:hiroyuki_sanadura', 'artist:shindo_l', 'artist:naokame', 'artist:kin_no_hiyoko', 'artist:masaru_yajiro', 'group:da_hootch', 'publisher:enshodo', 'language:english', 'anal', 'b*****b', 'oppai', 'glasses', 'stockings', 'group', 'nurse', 'hentai', 'ahegao', 'creampie', 'uncensored', 'genderbend', 'doujin', ], comment= "Takumi's life as a girl only continues to get more wild, as he (she?) continues to fall deeper into a life of promiscuity, drugs and unprotected sex with strangers. Will his friend Ryou be able to pull him out of this terrible spiral?", thumbnail_url= 'https://t.fakku.net/images/manga/t/tsf-story-append-20-english_1497401155_1502575464/thumbs/001.thumb.jpg', ) self.assertEqual(data, expected_data)
def get_values_from_gallery_link(self, link: str) -> Optional[GalleryData]: response = request_with_retries(link, { 'headers': self.settings.requests_headers, 'timeout': self.settings.timeout_timer, }, post=False, logger=self.logger) if not response: self.logger.error("Got no response from: {}".format(link)) return None response.encoding = 'utf-8' match_string = re.compile(constants.main_page + r"/view/(\d+)/*$") tags = [] soup = BeautifulSoup(response.text, 'html.parser') content_container = soup.find_all("div", class_="container")[0] if not content_container: self.logger.error("Could not find content container") return None is_doujinshi = False artists_container = content_container.find_all( "a", href=re.compile('/?q=artist:.*$')) for artist in artists_container: tags.append("artist:{}".format(artist.get_text())) language_container = content_container.find_all( "a", href=re.compile('/?q=language:.*$')) for language in language_container: tags.append("language:{}".format(language.get_text())) magazine_container = content_container.find_all( "a", href=re.compile('/?q=magazine:.*$')) for magazine in magazine_container: tags.append("magazine:{}".format(magazine.get_text())) parody_container = content_container.find_all( "a", href=re.compile('/?q=parody:.*$')) for parody in parody_container: tags.append("parody:{}".format(parody.get_text())) publisher_container = content_container.find_all( "a", href=re.compile('/?q=publisher:.*$')) for publisher in publisher_container: tags.append("publisher:{}".format(publisher.get_text())) tags_container = content_container.find_all( "a", href=re.compile('/?q=tag:.*$')) for tag in tags_container: tag_cleaned = tag.get_text().replace("\t", "").replace("\n", "") tags.append(tag_cleaned) if tag_cleaned == 'doujin': is_doujinshi = True thumbnail_url = soup.find("meta", property="og:image").get('content') match_result = match_string.match( soup.find("meta", property="og:url").get('content')) if not match_result: self.logger.error("Could not find gallery info container") return None gallery_id = match_result.group(1) gallery = GalleryData( gallery_id, link=link, title=content_container.find("h1", class_="title").get_text(), thumbnail_url=thumbnail_url, provider=self.name, posted=None, filesize=0, expunged=False, tags=translate_tag_list(tags), ) table_container = content_container.find("table", class_="view-page-details") if table_container: tr_container = table_container.find_all("tr") for tr in tr_container: if isinstance(tr, bs4.element.Tag): td_container = tr.find_all("td") is_description = False is_pages = False for td in td_container: if is_description: gallery.comment = td.get_text().replace( "\t", "").replace("\n", "") is_description = False if isinstance(td, bs4.element.Tag) and td.get_text( ) == 'Description': is_description = True if is_pages: right_text = td.get_text().replace("\t", "").replace( "\n", "") m = re.search(r'(\d+)', right_text) if m: gallery.filecount = int(m.group(1)) is_pages = False if isinstance( td, bs4.element.Tag) and td.get_text() == 'Pages': is_pages = True gallery.archiver_key = "{}/zip/{}".format(constants.main_page, gallery_id) if is_doujinshi: gallery.category = 'Doujinshi' else: gallery.category = 'Manga' return gallery
def get_values_from_gallery_link_json(self, link: str) -> Optional[GalleryData]: match_string = re.compile(constants.main_page + '/(.+)/$') m = match_string.match(link) if m: gallery_slug = m.group(1) else: return None api_link = constants.posts_api_url request_dict = construct_request_dict(self.settings, self.own_settings) request_dict['params'] = {'slug': gallery_slug} response = request_with_retries( api_link, request_dict, post=False, ) if not response: return None response.encoding = 'utf-8' try: response_data = response.json() except (ValueError, KeyError): logger.error( "Could not parse response to JSON: {}".format(api_link)) return None tags = [] thumbnail_url = '' if len(response_data) < 1: return None api_gallery = response_data[0] soup = BeautifulSoup(api_gallery['content']['rendered'], 'html.parser') artists_container = soup.find_all("a", href=re.compile(constants.main_page + '/artist/.*/$')) for artist in artists_container: tags.append("artist:{}".format(artist.get_text())) tags_container = soup.find_all("a", href=re.compile(constants.main_page + '/tag/.*/$')) for tag in tags_container: tags.append(tag.get_text()) thumbnail_small_container = soup.find("img") if thumbnail_small_container: thumbnail_url = thumbnail_small_container.get('src') gallery = GalleryData( gallery_slug, self.name, link=link, title=unescape(api_gallery['title']['rendered']), comment='', thumbnail_url=thumbnail_url, category='Manga', uploader='', posted=datetime.strptime(api_gallery['date_gmt'] + '+0000', "%Y-%m-%dT%H:%M:%S%z"), filecount=0, filesize=0, expunged=False, rating='', tags=translate_tag_list(tags), content=api_gallery['content']['rendered'], ) return gallery
def crawl_json(self, json_string: str, wanted_filters: QuerySet = None, wanted_only: bool = False) -> None: if not self.settings.gallery_model: return dict_list = [] json_decoded = json.loads(json_string) if type(json_decoded) == dict: dict_list.append(json_decoded) elif type(json_decoded) == list: dict_list = json_decoded galleries_gids = [] found_galleries = set() total_galleries_filtered: List[GalleryData] = [] gallery_wanted_lists: Dict[str, List['WantedGallery']] = defaultdict(list) for gallery in dict_list: galleries_gids.append(gallery['gid']) gallery['posted'] = datetime.fromtimestamp(int(gallery['posted']), timezone.utc) gallery_data = GalleryData(**gallery) total_galleries_filtered.append(gallery_data) for galleries_gid_group in list(chunks(galleries_gids, 900)): for found_gallery in self.settings.gallery_model.objects.filter( gid__in=galleries_gid_group): discard_approved, discard_message = self.discard_gallery_by_internal_checks( gallery=found_gallery, link=found_gallery.get_link()) if discard_approved: self.logger.info(discard_message) found_galleries.add(found_gallery.gid) for count, gallery in enumerate(total_galleries_filtered): if gallery.gid in found_galleries: continue if self.general_utils.discard_by_tag_list(gallery.tags): self.logger.info( "Gallery {} of {}: Skipping gallery {}, because it's tagged with global discarded tags" .format(count, len(total_galleries_filtered), gallery.title)) continue if wanted_filters: self.compare_gallery_with_wanted_filters( gallery, gallery.link, wanted_filters, gallery_wanted_lists) if wanted_only and not gallery_wanted_lists[gallery.gid]: continue self.logger.info( "Gallery {} of {}: Gallery {} will be processed.".format( count, len(total_galleries_filtered), gallery.title)) if gallery.thumbnail: original_thumbnail_url = gallery.thumbnail_url gallery.thumbnail_url = gallery.thumbnail gallery_instance = self.settings.gallery_model.objects.update_or_create_from_values( gallery) gallery_instance.thumbnail_url = original_thumbnail_url gallery_instance.save() else: self.settings.gallery_model.objects.update_or_create_from_values( gallery)
def get_values_from_gallery_link(self, link: str) -> Optional[GalleryData]: response = request_with_retries(link, { 'headers': self.settings.requests_headers, 'timeout': self.settings.timeout_timer, 'cookies': self.own_settings.cookies }, post=False, logger=self.logger) if not response: return None response.encoding = 'utf-8' new_text = re.sub(r'(<div class="right">\d+?)</b>', r'\1', response.text) soup = BeautifulSoup(new_text, 'html.parser') gallery_container = soup.find("div", class_=re.compile("content-wrap")) if gallery_container: gallery = GalleryData( link.replace(constants.main_url + '/', '').replace('manga/', 'hentai/')) gallery.link = link gallery.tags = [] gallery.provider = self.name gallery.title = gallery_container.find( "div", class_="content-name").h1.get_text() if gallery.gid.startswith('manga') or gallery.gid.startswith( 'hentai'): gallery.category = 'Manga' elif gallery.gid.startswith('doujinshi'): gallery.category = 'Doujinshi' thumbnail_container = gallery_container.find("img", class_="tablet-50") if thumbnail_container: gallery.thumbnail_url = thumbnail_container.get("src") if gallery.thumbnail_url and gallery.thumbnail_url.startswith( '//'): gallery.thumbnail_url = 'https:' + gallery.thumbnail_url is_doujinshi = False for gallery_row in gallery_container.find_all( "div", {"class": "row"}): left_text = gallery_row.find("div", { "class": "row-left" }).get_text() right_div = gallery_row.find("div", {"class": "row-right"}) if left_text == "Series" or left_text == "Parody": right_text = right_div.get_text() # if not right_text == "Original Work": gallery.tags.append(translate_tag("parody:" + right_text)) elif left_text == "Artist": for artist in right_div.find_all("a"): gallery.tags.append( translate_tag("artist:" + artist.get_text())) elif left_text == "Magazine": gallery.tags.append( translate_tag("magazine:" + right_div.get_text())) elif left_text == "Publisher": gallery.tags.append( translate_tag("publisher:" + right_div.get_text())) elif left_text == "Circle": gallery.tags.append( translate_tag("group:" + right_div.get_text())) elif left_text == "Event": gallery.tags.append( translate_tag("event:" + right_div.get_text())) elif left_text == "Book": belongs_to_container = right_div.find("a") if belongs_to_container: gallery.gallery_container_gid = belongs_to_container.get( "href")[1:] elif left_text == "Language": gallery.tags.append( translate_tag("language:" + right_div.get_text())) elif left_text == "Pages": right_text = right_div.get_text() m = re.search(r'^(\d+)', right_text) if m: gallery.filecount = int(m.group(1)) elif left_text == "Uploader": gallery.uploader, right_date_text = right_div.get_text( ).split(' on ') right_date_text = re.sub(r'(\d+)(st|nd|rd|th)', r'\1', right_date_text) gallery.posted = datetime.strptime(right_date_text, "%B %d, %Y") elif left_text == "Description": gallery.comment = right_div.get_text() elif left_text == "Tags": for tag_a in right_div.find_all( "a", href=lambda x: x and '/tags/' in x): if tag_a.get_text() == 'doujin': is_doujinshi = True gallery.tags.append(translate_tag(tag_a.get_text())) if is_doujinshi: gallery.category = 'Doujinshi' else: gallery.category = 'Manga' else: return None return gallery
def convert_api_response_text_to_gallery_dicts(text: str) -> list[GalleryData]: galleries: list[GalleryData] = [] # Based on: https://www.doujinshi.org/API_MANUAL.txt xml_root = ElementTree.fromstring(text) error = xml_root.find('ERROR') if error: return galleries for book in xml_root.findall('BOOK'): book_id = book.get('ID') if book_id is None: continue integer_id = int(book_id.replace('B', '')) gallery = GalleryData('mugi-' + book_id, constants.provider_name) gallery.link = constants.main_page + '/book/' + str(integer_id) gallery.tags = [] found_en_title = book.find('NAME_EN') if found_en_title is not None: gallery.title = found_en_title.text or '' found_jp_title = book.find('NAME_JP') if found_jp_title is not None: gallery.title_jpn = found_jp_title.text or '' gallery.comment = '' gallery.category = '' gallery.filesize = 0 found_data_pages = book.find('DATA_PAGES') if found_data_pages is not None and found_data_pages.text: gallery.filecount = int(found_data_pages.text) else: gallery.filecount = 0 gallery.uploader = '' gallery.thumbnail_url = 'https://img.doujinshi.org/big/{}/{}.jpg'.format( int(integer_id / 2000), integer_id) found_user = xml_root.find('USER') if found_user is not None: found_user_queries = found_user.find('Queries') if found_user_queries is not None and found_user_queries.text: gallery.queries = int(found_user_queries.text) else: gallery.queries = 0 else: gallery.queries = 0 # Check if we get the 0000-00-00 date found_date_released = book.find('DATE_RELEASED') if found_date_released is not None and found_date_released.text: date_components = found_date_released.text.split("-") if len(date_components) >= 3: if date_components[0] != '0000' and date_components[ 1] != '00' and date_components[2] != '00': gallery.posted = datetime.strptime( found_date_released.text + ' +0000', '%Y-%m-%d %z') found_links = book.find('LINKS') if found_links is not None: for item in found_links: item_type = item.get('TYPE') item_name_en = item.find('NAME_EN') if item_type == 'author': item_type = 'artist' elif item_type == 'circle': item_type = 'group' elif item_type == 'type' and item_name_en is not None: gallery.category = item_name_en.text continue elif item_type is None or item_type == '': if item_name_en is not None and not ( item_name_en.text == '' or item_name_en.text is None): gallery.tags.append(translate_tag(item_name_en.text)) continue if item_name_en is not None and not ( item_name_en.text == '' or item_name_en.text is None): gallery.tags.append( translate_tag(item_type + ":" + item_name_en.text)) found_data_language = book.find('DATA_LANGUAGE') if found_data_language is not None and not ( found_data_language.text == '' or found_data_language.text is None): gallery.tags.append( translate_tag( "language:" + translate_language_code(found_data_language.text))) # Add non-h as a tag. found_age = book.find('DATA_AGE') if found_age is not None and not ( found_age.text == '' or found_age.text is None) and found_age.text == '0': gallery.tags.append(translate_tag('non-h')) galleries.append(gallery) return galleries
def process_regular_gallery_page( self, link: str, response_text: str) -> Optional[GalleryData]: soup = BeautifulSoup(response_text, 'html.parser') gallery_container = soup.find("div", class_=re.compile("content-wrap")) if gallery_container: gallery = GalleryData( link.replace(constants.main_url + '/', '').replace('manga/', 'hentai/'), self.name) gallery.link = link gallery.tags = [] if self.own_settings.get_posted_date_from_feed: gallery.posted = self.parse_posted_date_from_feed( constants.aux_feed_url, gallery.gid) gallery.title = gallery_container.find( "div", class_="content-name").h1.get_text() if gallery.gid.startswith('manga') or gallery.gid.startswith( 'hentai'): gallery.category = 'Manga' elif gallery.gid.startswith('doujinshi'): gallery.category = 'Doujinshi' thumbnail_container = gallery_container.find("img", class_="tablet-50") if thumbnail_container: gallery.thumbnail_url = thumbnail_container.get("src") if gallery.thumbnail_url and gallery.thumbnail_url.startswith( '//'): gallery.thumbnail_url = 'https:' + gallery.thumbnail_url is_doujinshi = False for gallery_row in gallery_container.find_all( "div", {"class": "row"}): left_text = gallery_row.find("div", { "class": "row-left" }).get_text() right_div = gallery_row.find("div", {"class": "row-right"}) if left_text == "Series" or left_text == "Parody": right_text = right_div.get_text().strip() # if not right_text == "Original Work": gallery.tags.append(translate_tag("parody:" + right_text)) elif left_text == "Artist": for artist in right_div.find_all("a"): gallery.tags.append( translate_tag("artist:" + artist.get_text().strip())) elif left_text == "Author": for author in right_div.find_all("a"): gallery.tags.append( translate_tag("author:" + author.get_text().strip())) elif left_text == "Magazine": gallery.tags.append( translate_tag("magazine:" + right_div.get_text().strip())) belongs_to_magazine = right_div.find("a") if belongs_to_magazine: gallery.magazine_gid = belongs_to_magazine.get( "href")[1:] elif left_text == "Publisher": gallery.tags.append( translate_tag("publisher:" + right_div.get_text().strip())) elif left_text == "Circle": gallery.tags.append( translate_tag("group:" + right_div.get_text().strip())) elif left_text == "Event": gallery.tags.append( translate_tag("event:" + right_div.get_text().strip())) elif left_text == "Book": belongs_to_container = right_div.find("a") if belongs_to_container: gallery.gallery_container_gid = belongs_to_container.get( "href")[1:] elif left_text == "Language": gallery.tags.append( translate_tag("language:" + right_div.get_text().strip())) elif left_text == "Pages": right_text = right_div.get_text().strip() m = re.search(r'^(\d+)', right_text) if m: gallery.filecount = int(m.group(1)) elif left_text == "Uploader": gallery.uploader, right_date_text = right_div.get_text( ).strip().split(' on ') right_date_text = re.sub(r'(\d+)(st|nd|rd|th)', r'\1', right_date_text) gallery.posted = datetime.strptime(right_date_text, "%B %d, %Y") elif left_text == "Description": gallery.comment = right_div.get_text() elif left_text == "Tags": for tag_a in right_div.find_all( "a", href=lambda x: x and '/tags/' in x): if tag_a.get_text().strip() == 'doujin': is_doujinshi = True gallery.tags.append( translate_tag(tag_a.get_text().strip())) if is_doujinshi: gallery.category = 'Doujinshi' else: gallery.category = 'Manga' return gallery else: return None
def process_magazine_page(self, link: str, response_text: str) -> Optional[GalleryData]: soup = BeautifulSoup(response_text, 'html.parser') magazine_container = soup.find("div", class_="wrap") comic_regex = re.compile("content-comic") if magazine_container: gid = link.replace(constants.main_url + '/', '') gallery = GalleryData(gid, self.name) if self.own_settings.get_posted_date_from_feed: gallery.posted = self.parse_posted_date_from_feed( constants.aux_feed_url, gallery.gid) gallery.link = link gallery.tags = [] gallery.magazine_chapters_gids = [] gallery.title = magazine_container.find( "a", itemprop="item", href=re.compile("/" + gid)).span.get_text() gallery.category = 'Manga' # We assume every magazine is commercial, and we keep using panda definition thumbnail_container = magazine_container.find( "img", class_="content-poster") if thumbnail_container: gallery.thumbnail_url = thumbnail_container.get("src") if gallery.thumbnail_url and gallery.thumbnail_url.startswith( '//'): gallery.thumbnail_url = 'https:' + gallery.thumbnail_url description_container = magazine_container.find( "p", class_=re.compile("attribute-description")) if description_container: comment_text = description_container.decode_contents().replace( "\n", "").replace("<br/>", "\n") comment_soup = BeautifulSoup(comment_text, 'html.parser') gallery.comment = comment_soup.get_text() chapters_container = magazine_container.find_all( "div", class_=comic_regex) tags_set = set() artists_set = set() for chapter_container in chapters_container: chapter_title_container = chapter_container.find( "a", class_="content-title") if chapter_title_container: # chapter_title = chapter_title_container.get_text() chapter_link = chapter_title_container.get('href').replace( constants.main_url + '/', '') chapter_gid = chapter_link[1:] if chapter_link[ 0] == '/' else chapter_link gallery.magazine_chapters_gids.append(chapter_gid) tags_container = chapter_container.find( "div", {"class": "tags"}) for tag_a in tags_container.find_all( "a", href=lambda x: x and '/tags/' in x): tags_set.add(translate_tag(tag_a.get_text().strip())) artist = chapter_container.find( "a", href=lambda x: x and '/artists/' in x) if artist: artists_set.add("artist:" + translate_tag(artist.get_text().strip())) gallery.tags = list(tags_set) gallery.tags.extend(list(artists_set)) return gallery else: return None
def process_regular_gallery_page( self, link: str, response_text: str) -> Optional[GalleryData]: soup = BeautifulSoup(response_text, 'html.parser') gallery = GalleryData(link.replace(constants.main_url + '/', ''), self.name) gallery.link = link gallery.tags = [] title_container = soup.find("meta", property="og:title") gallery.title = title_container['content'] if title_container else '' # Defaulting to Doujinshi (might need to change later) gallery.category = 'Doujinshi' description_container = soup.find("script", id="ProductJson-product-template") # We can get most data from this object parsed_json = json.loads(description_container.string) # gallery.title = parsed_json.get('title', '') gallery.posted = date_parser.parse( parsed_json.get('published_at', None)) thumbnail_url = parsed_json.get('featured_image', None) if thumbnail_url: gallery.thumbnail_url = urljoin(thumbnail_url, urlparse(thumbnail_url).path) if gallery.thumbnail_url and gallery.thumbnail_url.startswith( '//'): gallery.thumbnail_url = 'https:' + gallery.thumbnail_url if 'vendor' in parsed_json: gallery.tags.append( translate_tag("artist:" + parsed_json['vendor'])) if 'tags' in parsed_json: for tag in parsed_json['tags']: gallery.tags.append(translate_tag(tag)) if 'description' in parsed_json: description_soup = BeautifulSoup(parsed_json['description'], 'html.parser') description_text = description_soup.get_text().replace( "Synopsis:", "") description_text_newline = description_soup.get_text("\n").replace( "Synopsis:", "") description_text_remove = description_text description_text_remove = self.PAGES_REGEX.sub( '', description_text_remove) description_text_remove = self.JP_TITLE_REGEX.sub( '', description_text_remove) description_text_remove = self.DATE_CONVENTION_REGEX.sub( '', description_text_remove) other_language_found = self.OTHER_LANGUAGE_REGEX.search( description_text_newline) if other_language_found: gallery.tags.append( translate_tag("language:" + other_language_found.group(1))) else: gallery.tags.append(translate_tag("language:english")) pages_text_found = self.PAGES_REGEX.search( description_text_newline) if pages_text_found: gallery.filecount = int(pages_text_found.group(1)) jp_text_found = self.JP_TITLE_REGEX.search( description_text_newline) if jp_text_found: gallery.title_jpn = jp_text_found.group(1) convention_text_found = self.DATE_CONVENTION_REGEX.search( description_text_newline) if convention_text_found: gallery.tags.append( translate_tag("event:" + convention_text_found.group(1))) reparsed_text = '' # roundabout way to remove all after the used regexs. for char_count, char in enumerate(description_text_remove): if description_text_remove[char_count] == description_text[ char_count]: reparsed_text += char else: break gallery.comment = reparsed_text.strip() return gallery