def getCookie(self): if self.checkCookie()[0]: return True, "Already logged in" self.log.info("Trying to avoid rate limiting!") time.sleep(5) self.log.info("Not logged in. Doing login.") home = "https://www.patreon.com/home" login = "******" try: soup = self.get_soup(home) except Exception as e: import pdb print(e) pdb.set_trace() print(e) # These won't work for the particular recaptcha flavor patreon uses. Sigh. if soup.find_all("div", class_="g-recaptcha"): soup = self.handle_recaptcha(soup, home, login) if soup.find_all("div", id="hcaptcha_widget"): soup = self.handle_hcaptcha(soup, home, login) if soup.find_all("div", class_="g-recaptcha"): self.log.error("Failed after attempting to solve recaptcha!") raise exceptions.NotLoggedInException( "Login failed due to recaptcha!") if soup.find_all("div", class_="hcaptcha_widget"): self.log.error("Failed after attempting to solve recaptcha!") raise exceptions.NotLoggedInException( "Login failed due to recaptcha!") # So patreon uses RavenJS, which does a bunch of really horrible change-watching crap on input # fields. I couldn't figure out how to properly fire the on-change events, so let's just # use the debug-protocol interface to type our login info in manually. self.cr.execute_javascript_function( "document.querySelector(\"input[id='email']\").focus()") for char in settings[self.pluginShortName]['username']: self.cr.Input_dispatchKeyEvent(type='char', text=char) self.cr.execute_javascript_statement( "document.querySelector(\"input[id='password']\").focus()") for char in settings[self.pluginShortName]['password']: self.cr.Input_dispatchKeyEvent(type='char', text=char) self.cr.execute_javascript_statement( "document.querySelector(\"button[type='submit']\").click()") content = self.cr.get_rendered_page_source() if not settings[self.pluginShortName]['username'] in content: raise exceptions.CannotAccessException("Could not log in?") self.wg.saveCookies() return self.checkCookie()
def _get_art_post(self, postId, artistName): post = self.get_api_json("/posts/{pid}".format(pid=postId) + "?include=media") post_content = post['data'] post_info = post_content['attributes'] if 'current_user_can_view' in post_info and post_info[ 'current_user_can_view'] is False: self.log.warning( "You apparently cannot view post %s for artist %s. Ignoring.", postId, artistName) fail = {'status': ''} return fail if not 'included' in post: self.log.warning( "No contents on post %s for artist %s (%s). Please report if this is in error.", postId, artistName, post_info['url']) fail = {'status': ''} return fail attachments = { item['id']: item for item in post['included'] if item['type'] == 'attachment' } media = { item['id']: item for item in post['included'] if item['type'] == 'media' } tags = [] if 'user_defined_tags' in post_content['relationships']: for tagmeta in post_content['relationships']['user_defined_tags'][ 'data']: tags.append(tagmeta['id'].split(";")[-1]) if 'current_user_can_view' in post_content and not post_content[ 'current_user_can_view']: raise exceptions.CannotAccessException( "You can't view that content!") # if not 'content' in post_info: pprint.pprint(post_content) ret = { 'page_desc': post_info['content'], 'page_title': post_info['title'], 'post_time': dateutil.parser.parse( post_info['published_at']).replace(tzinfo=None), 'post_tags': tags, 'post_embeds': [], } # print("Post:") # pprint.pprint(post) # print("Content:") # pprint.pprint(post_info['content']) files = [] try: if "post_file" in post_info and post_info['post_file']: furl = urllib.parse.unquote(post_info['post_file']['url']) # print("Post file!", post_info['post_file']['url'], furl) fpath = self.save_image(artistName, postId, post_info['post_file']['name'], furl) files.append(fpath) if 'post_type' in post_info and post_info[ 'post_type'] == 'video_embed': # print("Post video_embed") fpath = self.fetch_video_embed(post_info) if fpath: files.append(fpath) ret['post_embeds'].append(post_info) for aid, dat_struct in attachments.items(): # print("Post attachments") fpath = self.save_attachment(artistName, postId, dat_struct) files.append(fpath) for aid, dat_struct in media.items(): # print("Post attachments") fpath = self.save_media(artistName, postId, dat_struct) files.append(fpath) if 'embed' in post_info and post_info['embed']: for item in self._handle_embed(post_info['embed']): files.append(fpath) ret['post_embeds'].append(post_info['embed']) except urllib.error.URLError: self.log.error("Failure retreiving content from post: %s", post) ctnt_soup = bs4.BeautifulSoup(post_info['content'], 'lxml') for img in ctnt_soup.find_all("img", src=True): furl = img['src'] fparsed = urllib.parse.urlparse(furl) fname = fparsed.path.split("/")[-1] fpath = self.save_image(artistName, postId, fname, furl) files.append(fpath) # Youtube etc are embedded as iframes. for ifr in ctnt_soup.find_all("iframe", src=True): ret['post_embeds'].append(ifr['src']) if len(files): self.log.info("Found %s images/attachments on post.", len(attachments)) else: self.log.warning("No images/attachments on post %s!", postId) files = [filen for filen in files if filen] ret['dl_path'] = files ret['status'] = 'Succeeded' # pprint.pprint(ret) return ret
def _getArtPage(self, dlPathBase, artPageUrl, artistName): soup = self.wg.getSoup(artPageUrl) soups = str(soup) if 'That submission has been deleted.' in soups: self.log.warning("Item %s has been deleted by the poster!", artPageUrl) return self.build_page_ret(status="Deleted", fqDlPath=None) if 'The owner has restricted this submission to members' in soups: raise exceptions.NotLoggedInException("Not logged in?") if "You may need to go to that member's userpage (use name link above) and request that they give you access" in soups: raise exceptions.CannotAccessException( "Friends only thingie (why is this even a thing)!") datespan = soup.find('span', id='submittime_exact') if not datespan: self.log.warning("No date-span in content on page '%s'!!", artPageUrl) return self.build_page_ret(status='Failed', pageTitle="Missing content!", fqDlPath=None) postTime = self._extractPostTimestamp(soup) postTags = self._extractPostTags(soup) titleBar, dummy_stats, dummy_footer = soup.body.find_all( 'div', class_='elephant_555753', recursive=False) story_div = None mainDivs = soup.body.find_all('div', class_='elephant_white', recursive=False) print("Maindivs: ", len(mainDivs)) if len(mainDivs) == 2: print("Sequence item!") imgDiv, desc_div = mainDivs footer = desc_div imageURL = [self._getContentUrlFromPage(imgDiv)] elif len(mainDivs) == 3 or len(mainDivs) == 4: if len(mainDivs) == 3: imgDiv, seqDiv, desc_div = mainDivs footer = desc_div if len(mainDivs) == 4: imgDiv, seqDiv, desc_div, footer = mainDivs imageURL = set() base_img = self._getContentUrlFromPage(imgDiv) if base_img: imageURL.add(base_img) for img in self._getSeqImageDivs(seqDiv): imageURL.add(img) for img in self._getSeqImageDivs(desc_div): imageURL.add(img) self.log.info("Found %s item series on page!", len(imageURL)) elif len(mainDivs) == 5: _header_div, img_div, story_div, desc_div, footer = mainDivs imageURL = set() base_img = self._getContentUrlFromPage(img_div) if base_img: imageURL.add(base_img) for img in self._getSeqImageDivs(img_div): imageURL.add(img) for img in self._getSeqImageDivs(story_div): imageURL.add(img) for img in self._getSeqImageDivs(desc_div): imageURL.add(img) self.log.info("Found %s item series on page!", len(imageURL)) else: soupp = str(mainDivs) with open("log.html", "w") as fp: fp.write(soupp) print(soupp) raise ValueError("Unknown number of mainDivs! %s" % len(mainDivs)) # print(imageURL) itemTitle = self._extractTitle(titleBar) itemCaption = self._extractDescription(story_div, desc_div, footer) if not imageURL: self.log.error("OH NOES!!! No image on page = " + artPageUrl) return self.build_page_ret(status="Failed", fqDlPath=None) imPaths = [] for image in imageURL: recPath = self._fetchImage(image, dlPathBase, itemCaption, itemTitle, artPageUrl) imPaths.append(recPath) return self.build_page_ret(status="Succeeded", fqDlPath=imPaths, pageDesc=itemCaption, pageTitle=itemTitle, postTags=postTags, postTime=postTime)