def loginUsingCookie(self, login_cookie=None): """ Log in to Pixiv using saved cookie, return True if success """ if login_cookie is None or len(login_cookie) == 0: login_cookie = self._config.cookie if len(login_cookie) > 0: PixivHelper.print_and_log('info', 'Trying to log in with saved cookie') self.clearCookie() self._loadCookie(login_cookie) res = self.open_with_retry('https://www.pixiv.net/') resData = res.read() parsed = BeautifulSoup(resData) PixivHelper.GetLogger().info('Logging in, return url: %s', res.geturl()) if "logout.php" in resData: PixivHelper.print_and_log('info', 'Login successful.') PixivHelper.GetLogger().info('Logged in using cookie') self.getMyId(parsed) temp_locale = str(res.geturl()).replace('https://www.pixiv.net/', '').replace('/', '') if len(temp_locale) > 0: self._locale = '/' + temp_locale PixivHelper.GetLogger().info('Locale = %s', self._locale) return True else: PixivHelper.GetLogger().info('Failed to log in using cookie') PixivHelper.print_and_log('info', 'Cookie already expired/invalid.') return False
def processLoginResult(self, response, username, password): PixivHelper.GetLogger().info('Logging in, return url: %s', response.geturl()) # check the returned json js = response.read() PixivHelper.GetLogger().info(str(js)) result = json.loads(js) # Fix Issue #181 if result["body"] is not None and result["body"].has_key("success"): for cookie in self._ua_handlers['_cookies'].cookiejar: if cookie.name == 'PHPSESSID': PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value)) self._config.cookie = cookie.value self._config.writeConfig(path=self._config.configFileLocation) break # check whitecube page = self.open_with_retry(result["body"]["success"]["return_to"]) parsed = BeautifulSoup(page) self.getMyId(parsed) # store the username and password in memory for oAuth login self._config.username = username self._config.password = password return True else: if result["body"] is not None and result["body"].has_key("validation_errors"): PixivHelper.print_and_log('info', "Server reply: " + str(result["body"]["validation_errors"])) if str(result["body"]["validation_errors"]).find("reCAPTCHA") > 0: print("Please follow the method described in https://github.com/Nandaka/PixivUtil2/issues/505") else: PixivHelper.print_and_log('info', 'Unknown login issue, please use cookie login method.') return False
def getImagePage(self, imageId, parent=None, fromBookmark=False, bookmark_count=-1, image_response_count=-1): image = None response = None PixivHelper.GetLogger().debug("Getting image page: {0}".format(imageId)) if self._isWhitecube: url = "https://www.pixiv.net/rpc/whitecube/index.php?mode=work_details_modal_whitecube&id={0}&tt={1}".format(imageId, self._whitecubeToken) response = self.open(url).read() PixivHelper.GetLogger().debug(response); image = PixivModelWhiteCube.PixivImage(imageId, response, parent, fromBookmark, bookmark_count, image_response_count, dateFormat=self._config.dateFormat) # overwrite artist info self.getMemberInfoWhitecube(image.artist.artistId, image.artist) else: url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(imageId) response = self.open(url).read() parsed = BeautifulSoup(response) image = PixivModel.PixivImage(imageId, parsed, parent, fromBookmark, bookmark_count, image_response_count, dateFormat=self._config.dateFormat) if image.imageMode == "ugoira_view" or image.imageMode == "bigNew": image.ParseImages(parsed) parsed.decompose() return (image, response)
def loginUsingCookie(self, login_cookie=None): """ Log in to Pixiv using saved cookie, return True if success """ if login_cookie is None or len(login_cookie) == 0: login_cookie = self._config.cookie if len(login_cookie) > 0: PixivHelper.print_and_log('info', 'Trying to log in with saved cookie') self._loadCookie(login_cookie) res = self.open('https://www.pixiv.net/mypage.php') resData = res.read() parsed = BeautifulSoup(resData) self.detectWhiteCube(parsed, res.geturl()) if "logout.php" in resData: PixivHelper.print_and_log('info', 'Login successful.') PixivHelper.GetLogger().info('Logged in using cookie') self.getMyId(parsed) return True else: PixivHelper.GetLogger().info('Failed to log in using cookie') PixivHelper.print_and_log('info', 'Cookie already expired/invalid.') return False
def processLoginResult(self, response): PixivHelper.GetLogger().info('Logging in, return url: ' + response.geturl()) # check the returned json js = response.read() PixivHelper.GetLogger().info(str(js)) result = json.loads(js) # Fix Issue #181 if result["body"] is not None and result["body"].has_key("success"): for cookie in self._ua_handlers['_cookies'].cookiejar: if cookie.name == 'PHPSESSID': PixivHelper.printAndLog('info', 'new cookie value: ' + str(cookie.value)) self._config.cookie = cookie.value self._config.writeConfig(path=self._config.configFileLocation) break # check whitecube page = self.open(result["body"]["success"]["return_to"]) parsed = BeautifulSoup(page) self.detectWhiteCube(parsed, page.geturl()) return True else: if result["body"] is not None and result["body"].has_key("validation_errors"): PixivHelper.printAndLog('info', "Server reply: " + str(result["body"]["validation_errors"])) else: PixivHelper.printAndLog('info', 'Unknown login issue, please use cookie login method.') return False
def getImagePage(self, image_id, parent=None, from_bookmark=False, bookmark_count=-1, image_response_count=-1): image = None response = None PixivHelper.GetLogger().debug("Getting image page: %s", image_id) url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format( image_id) # response = self.open(url).read() response = self.getPixivPage(url, returnParsed=False).read() self.handleDebugMediumPage(response, image_id) # Issue #355 new ui handler image = None try: if response.find("globalInitData") > 0: PixivHelper.print_and_log('debug', 'New UI Mode') # Issue #420 _tzInfo = None if self._config.useLocalTimezone: _tzInfo = PixivHelper.LocalUTCOffsetTimezone() image = PixivModelWhiteCube.PixivImage( image_id, response, parent, from_bookmark, bookmark_count, image_response_count, dateFormat=self._config.dateFormat, tzInfo=_tzInfo) if image.imageMode == "ugoira_view": ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format( image_id) meta_response = self.open_with_retry( ugoira_meta_url).read() image.ParseUgoira(meta_response) if parent is None: if from_bookmark: image.originalArtist.reference_image_id = image_id self.getMemberInfoWhitecube( image.originalArtist.artistId, image.originalArtist) else: image.artist.reference_image_id = image_id self.getMemberInfoWhitecube(image.artist.artistId, image.artist) except: PixivHelper.GetLogger().error("Respose data: \r\n %s", response) raise return (image, response)
def parseList(filename, rootDir=None): '''read list.txt and return the list of PixivListItem''' l = list() if not os.path.exists(filename) : raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION) reader = PixivHelper.OpenTextFile(filename) lineNo = 1 try: for line in reader: originalLine = line ##PixivHelper.safePrint("Processing: " + line) if line.startswith('#') or len(line) < 1: continue if len(line.strip()) == 0: continue line = PixivHelper.toUnicode(line) line = line.strip() items = line.split(" ", 1) member_id = int(items[0]) path = "" if len(items) > 1: path = items[1].strip() path = path.replace('\"', '') if rootDir != None: path = path.replace('%root%', rootDir) else: path = path.replace('%root%', '') path = os.path.abspath(path) # have drive letter if re.match(r'[a-zA-Z]:', path): dirpath = path.split(os.sep, 1) dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None) path = os.sep.join(dirpath) else: path = PixivHelper.sanitizeFilename(path, rootDir) path = path.replace('\\\\', '\\') path = path.replace('\\', os.sep) listItem = PixivListItem(member_id, path) l.append(listItem) lineNo = lineNo + 1 originalLine = "" except UnicodeDecodeError: PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'.format(originalLine, lineNo)) except: PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo)) reader.close() return l
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False): ''' get artist information using Ajax and AppAPI ''' try: url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format( member_id) info = self.get_from_cache(url) if info is None: PixivHelper.GetLogger().debug("Getting member information: %s", member_id) infoStr = self.open(url).read() info = json.loads(infoStr) self.put_to_cache(url, info) artist.ParseInfo(info, False, bookmark=bookmark) # will throw HTTPError if user is suspended/not logged in. url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id) info_ajax = self.get_from_cache(url_ajax) if info_ajax is None: info_ajax_str = self.open(url_ajax).read() info_ajax = json.loads(info_ajax_str) self.put_to_cache(url_ajax, info_ajax) # 2nd pass to get the background artist.ParseBackground(info_ajax) return artist except urllib2.HTTPError, error: errorCode = error.getcode() errorMessage = error.get_data() PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage) payload = demjson.decode(errorMessage) # Issue #432 if payload.has_key("message"): msg = payload["message"] elif payload.has_key("error") and payload["error"] is not None: msgs = list() msgs.append(payload["error"]["user_message"]) msgs.append(payload["error"]["message"]) msgs.append(payload["error"]["reason"]) msg = ",".join(msgs) if errorCode == 401: raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage) elif errorCode == 403: raise PixivException( msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage) else: raise PixivException( msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)
def ParseWorksData(self, page): temp = page.find(attrs={'class': 'meta'}).findAll('li') #07/22/2011 03:09|512×600|RETAS STUDIO #07/26/2011 00:30|Manga 39P|ComicStudio 鉛筆 つけペン #1/05/2011 07:09|723×1023|Photoshop SAI [ R-18 ] #2013年3月16日 06:44 | 800×1130 | Photoshop ComicStudio | R-18 #2013年12月14日 19:00 855×1133 PhotoshopSAI self.worksDate = PixivHelper.toUnicode(temp[0].string, encoding=sys.stdin.encoding) if self.dateFormat is not None and len( self.dateFormat) > 0 and '%' in self.dateFormat: # use the user defined format try: self.worksDateDateTime = datetime.strptime( self.worksDate, self.dateFormat) except ValueError as ve: PixivHelper.GetLogger().exception( 'Error when parsing datetime: {0} for imageId {1} using date format {2}' .format(self.worksDate, str(self.imageId), str(self.dateFormat)), ve) raise else: self.worksDate = self.worksDate.replace(u'/', u'-') if self.worksDate.find('-') > -1: try: self.worksDateDateTime = datetime.strptime( self.worksDate, u'%m-%d-%Y %H:%M') except ValueError as ve: PixivHelper.GetLogger().exception( 'Error when parsing datetime: {0} for imageId {1}'. format(self.worksDate, self.imageId), ve) self.worksDateDateTime = datetime.strptime( self.worksDate.split(" ")[0], u'%Y-%m-%d') else: tempDate = self.worksDate.replace(u'年', '-').replace( u'月', '-').replace(u'日', '') self.worksDateDateTime = datetime.strptime( tempDate, '%Y-%m-%d %H:%M') self.worksResolution = unicode(temp[1].string).replace(u'×', u'x') toolsTemp = page.find(attrs={ 'class': 'meta' }).find(attrs={'class': 'tools'}) if toolsTemp != None and len(toolsTemp) > 0: tools = toolsTemp.findAll('li') for tool in tools: self.worksTools = self.worksTools + ' ' + unicode(tool.string) self.worksTools = self.worksTools.strip()
class TestPixivModel_WhiteCube(unittest.TestCase): currPath = unicode(os.path.abspath('.')) PixivHelper.GetLogger() def testParseLoginForm(self): p = open('./test/pixiv-whitecube-main.html', 'r') page = BeautifulSoup(p.read()) init_config = page.find('input', attrs={'id': 'init-config'}) js_init_config = json.loads(init_config['value']) self.assertIsNotNone(js_init_config) self.assertIsNotNone(js_init_config["pixiv.context.token"]) def testParseImage(self): p = open('./test/work_details_modal_whitecube.json', 'r') image = PixivImage(59521621, p.read()) self.assertIsNotNone(image) image.PrintInfo() self.assertEqual(image.imageMode, "big") def testParseManga(self): p = open('./test/work_details_modal_whitecube-manga.json', 'r') image = PixivImage(59532028, p.read()) self.assertIsNotNone(image) image.PrintInfo() self.assertEqual(image.imageMode, "manga")
def WriteInfo(self, filename): info = None try: info = codecs.open(filename, 'wb', encoding='utf-8') except IOError: info = codecs.open(str(self.imageId) + ".txt", 'wb', encoding='utf-8') PixivHelper.GetLogger().exception( "Error when saving image info: " + filename + ", file is saved to: " + str(self.imageId) + ".txt") info.write("ArtistID = " + str(self.artist.artistId) + "\r\n") info.write("ArtistName = " + self.artist.artistName + "\r\n") info.write("ImageID = " + str(self.imageId) + "\r\n") info.write("Title = " + self.imageTitle + "\r\n") info.write("Caption = " + self.imageCaption + "\r\n") info.write("Tags = " + ", ".join(self.imageTags) + "\r\n") info.write("Image Mode = " + self.imageMode + "\r\n") info.write("Pages = " + str(self.imageCount) + "\r\n") info.write("Date = " + self.worksDate + "\r\n") info.write("Resolution = " + self.worksResolution + "\r\n") info.write("Tools = " + self.worksTools + "\r\n") info.write("BookmarkCount= " + str(self.bookmark_count) + "\r\n") info.write( "Link = http://www.pixiv.net/member_illust.php?mode=medium&illust_id=" + str(self.imageId) + "\r\n") info.write("Ugoira Data= " + str(self.ugoira_data) + "\r\n") info.close()
def WriteJSON(self, filename): info = None try: info = codecs.open(filename, 'w', encoding='utf-8') except IOError: info = codecs.open(str(self.imageId) + ".txt", 'w', encoding='utf-8') PixivHelper.GetLogger().exception( "Error when saving image info: " + filename + ", file is saved to: " + str(self.imageId) + ".txt") info.write("{" + "\r\n") info.write("\t" + json.dumps("Artist ID") + ": " + json.dumps(self.artist.artistId, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Artist Name") + ": " + json.dumps(self.artist.artistName, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Image ID") + ": " + json.dumps(self.imageId, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Title") + ": " + json.dumps(self.imageTitle, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Caption") + ": " + json.dumps(self.imageCaption, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Tags") + ": " + json.dumps(self.imageTags, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Image Mode") + ": " + json.dumps(self.imageMode, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Pages") + ": " + json.dumps(self.imageCount, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Date") + ": " + json.dumps(self.worksDate, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Resolution") + ": " + json.dumps(self.worksResolution, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Tools") + ": " + json.dumps(self.worksTools, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("BookmarkCount") + ": " + json.dumps(self.bookmark_count, ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Link") + ": " + json.dumps( "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=" + str(self.imageId), ensure_ascii=False) + "," + "\r\n") info.write("\t" + json.dumps("Ugoira Data") + ": " + json.dumps(self.ugoira_data, ensure_ascii=False) + "\r\n") if len(self.descriptionUrlList) > 0: info.write( "\t" + json.dumps("Urls") + ": " + json.dumps(self.descriptionUrlList, ensure_ascii=False) + "," + "\r\n") info.write("}") info.close()
def processLoginResult(self, response): PixivHelper.GetLogger().info('Logging in, return url: ' + response.geturl()) ## failed login will return to either of these page: ## http://www.pixiv.net/login.php ## https://www.secure.pixiv.net/login.php if response.geturl().find('pixiv.net/login.php') == -1: PixivHelper.printAndLog('info', 'Logged in') ## write back the new cookie value for cookie in self._ua_handlers['_cookies'].cookiejar: if cookie.name == 'PHPSESSID': PixivHelper.printAndLog( 'info', 'new cookie value: ' + str(cookie.value)) self._config.cookie = cookie.value self._config.writeConfig( path=self._config.configFileLocation) break return True else: errors = self.parseLoginError(response) if len(errors) > 0: for error in errors: if error.string is not None: PixivHelper.printAndLog( 'error', 'Server Reply: ' + error.string) else: PixivHelper.printAndLog('info', 'Wrong username or password.') return False
def WriteInfo(self, filename): info = None try: # Issue #421 ensure subdir exists. PixivHelper.makeSubdirs(filename) info = codecs.open(filename, 'wb', encoding='utf-8') except IOError: info = codecs.open(str(self.imageId) + ".txt", 'wb', encoding='utf-8') PixivHelper.GetLogger().exception("Error when saving image info: %s, file is saved to: %s.txt", filename, self.imageId) info.write(u"ArtistID = {0}\r\n".format(self.parent.artistId)) info.write(u"ArtistName = {0}\r\n".format(self.parent.artistName)) info.write(u"ImageID = {0}\r\n".format(self.imageId)) info.write(u"Title = {0}\r\n".format(self.imageTitle)) info.write(u"Caption = {0}\r\n".format(self.body_text)) # info.write(u"Tags = " + ", ".join(self.imageTags) + "\r\n") if self.is_restricted: info.write(u"Image Mode = {0}, Restricted\r\n".format(self.type)) else: info.write(u"Image Mode = {0}\r\n".format(self.type)) info.write(u"Pages = {0}\r\n".format(self.imageCount)) info.write(u"Date = {0}\r\n".format(self.worksDate)) # info.write(u"Resolution = " + self.worksResolution + "\r\n") # info.write(u"Tools = " + self.worksTools + "\r\n") info.write(u"Like Count = {0}\r\n".format(self.likeCount)) info.write(u"Link = https://www.pixiv.net/fanbox/creator/{0}/post/{1}\r\n".format(self.parent.artistId, self.imageId)) # info.write("Ugoira Data = " + str(self.ugoira_data) + "\r\n") if len(self.embeddedFiles) > 0: info.write("Urls =\r\n") for link in self.embeddedFiles: info.write(" - {0}\r\n".format(link)) info.close()
def _configureBrowser(self, config): if config is None: PixivHelper.GetLogger().info("No config given") return global defaultConfig if defaultConfig is None: defaultConfig = config self._config = config if config.useProxy: if config.proxyAddress.startswith('socks'): parseResult = urlparse.urlparse(config.proxyAddress) assert parseResult.scheme and parseResult.hostname and parseResult.port socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4 socks.setdefaultproxy(socksType, parseResult.hostname, parseResult.port) socks.wrapmodule(urllib) socks.wrapmodule(urllib2) socks.wrapmodule(httplib) PixivHelper.GetLogger().info("Using SOCKS Proxy: %s", config.proxyAddress) else: self.set_proxies(config.proxy) PixivHelper.GetLogger().info("Using Proxy: %s", config.proxyAddress) # self.set_handle_equiv(True) # self.set_handle_gzip(True) self.set_handle_redirect(True) self.set_handle_referer(True) self.set_handle_robots(False) self.set_debug_http(config.debugHttp) if config.debugHttp: PixivHelper.GetLogger().info('Debug HTTP enabled.') # self.visit_response self.addheaders = [('User-agent', config.useragent)] # force utf-8, fix issue #184 self.addheaders = [('Accept-Charset', 'utf-8')] socket.setdefaulttimeout(config.timeout)
def WriteUgoiraData(self, filename): info = None try: info = codecs.open(filename, 'wb', encoding='utf-8') except IOError: info = codecs.open(str(self.imageId) + ".js", 'wb', encoding='utf-8') PixivHelper.GetLogger().exception("Error when saving image info: " + filename + ", file is saved to: " + str(self.imageId) + ".js") info.write(str(self.ugoira_data)) info.close()
def login_with_username_and_password(self): PixivHelper.GetLogger().info( "Login to OAuth using username and password.") oauth_response = requests.post(self._url, self._get_values_for_login(), headers=self._get_default_headers(), proxies=self._proxies, verify=self._validate_ssl) return oauth_response
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None): artist = None response = None if tags is not None: tags = PixivHelper.encode_tags(tags) else: tags = '' limit = 48 offset = (page - 1) * limit need_to_slice = False if bookmark: # https://www.pixiv.net/ajax/user/1039353/illusts/bookmarks?tag=&offset=0&limit=24&rest=show url = 'https://www.pixiv.net/ajax/user/{0}/illusts/bookmarks?tag={1}&offset={2}&limit={3}&rest=show'.format( member_id, tags, offset, limit) else: # https://www.pixiv.net/ajax/user/1813972/illusts/tag?tag=Fate%2FGrandOrder?offset=0&limit=24 # https://www.pixiv.net/ajax/user/1813972/manga/tag?tag=%E3%83%A1%E3%82%A4%E3%82%AD%E3%83%B3%E3%82%B0?offset=0&limit=24 # https://www.pixiv.net/ajax/user/5238/illustmanga/tag?tag=R-18&offset=0&limit=48 # https://www.pixiv.net/ajax/user/1813972/profile/all url = None if len(tags) > 0: url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format( member_id, tags, offset, limit) elif self._config.r18mode: url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format( member_id, 'R-18', offset, limit) else: url = 'https://www.pixiv.net/ajax/user/{0}/profile/all'.format( member_id) need_to_slice = True PixivHelper.print_and_log('info', 'Member Url: ' + url) if url is not None: # cache the response response = self._get_from_cache(url) if response is None: try: response = self.open_with_retry(url).read() except urllib.error.HTTPError as ex: if ex.code == 404: response = ex.read() self._put_to_cache(url, response) PixivHelper.GetLogger().debug(response) artist = PixivModelWhiteCube.PixivArtist(member_id, response, False, offset, limit) artist.reference_image_id = artist.imageList[0] if len( artist.imageList) > 0 else 0 self.getMemberInfoWhitecube(member_id, artist, bookmark) if artist.haveImages and need_to_slice: artist.imageList = artist.imageList[offset:offset + limit] return (artist, response)
def __init__(self, config, cookie_jar): # fix #218 try: mechanize.Browser.__init__(self, factory=mechanize.RobustFactory()) except BaseException: PixivHelper.GetLogger().info("Using default factory (mechanize 3.x ?)") mechanize.Browser.__init__(self) self._configureBrowser(config) self._configureCookie(cookie_jar)
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False): ''' get artist information using AppAPI ''' url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id) if self._cache.has_key(url): info = self._cache[url] else: PixivHelper.GetLogger().debug("Getting member information: {0}".format(member_id)) infoStr = self.open(url).read() info = json.loads(infoStr) self._cache[url] = info artist.ParseInfo(info, False, bookmark=bookmark)
def loginUsingCookie(self, loginCookie=None): """ Log in to Pixiv using saved cookie, return True if success """ if loginCookie is None or len(loginCookie) == 0: loginCookie = self._config.cookie if len(loginCookie) > 0: PixivHelper.printAndLog('info', 'Trying to log with saved cookie') self._loadCookie(loginCookie) req = self._makeRequest('http://www.pixiv.net/mypage.php') self.open(req) res_url = self.response().geturl() if res_url == 'http://www.pixiv.net/mypage.php': PixivHelper.printAndLog('info', 'Login successfull.') PixivHelper.GetLogger().info('Logged in using cookie') return True else: PixivHelper.GetLogger().info('Failed to login using cookie, returned page: ' + res_url) PixivHelper.printAndLog('info', 'Cookie already expired/invalid.') return False
def configureBrowser(browser, config): if config == None: PixivHelper.GetLogger().info("No config given") return global defaultConfig if defaultConfig == None: defaultConfig = config if config.useProxy: if config.proxyAddress.startswith('socks'): parseResult = urlparse.urlparse(config.proxyAddress) assert parseResult.scheme and parseResult.hostname and parseResult.port socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4 socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, parseResult.hostname, parseResult.port) socks.wrapmodule(urllib) socks.wrapmodule(urllib2) socks.wrapmodule(httplib) PixivHelper.GetLogger().info("Using SOCKS Proxy: " + config.proxyAddress) else: browser.set_proxies(config.proxy) PixivHelper.GetLogger().info("Using Proxy: " + config.proxyAddress) browser.set_handle_equiv(True) #browser.set_handle_gzip(True) browser.set_handle_redirect(True) browser.set_handle_referer(True) browser.set_handle_robots(config.useRobots) browser.set_debug_http(config.debugHttp) if config.debugHttp: PixivHelper.GetLogger().info('Debug HTTP enabled.') browser.visit_response browser.addheaders = [('User-agent', config.useragent)] socket.setdefaulttimeout(config.timeout)
def CreateUgoira(self, filename): if len(self.ugoira_data) == 0: PixivHelper.GetLogger().exception("Missing ugoira animation info for image: " + str(self.imageId)) zipTarget = filename[:-4] + ".ugoira" if os.path.exists(zipTarget): os.remove(zipTarget) shutil.copyfile(filename, zipTarget) zipSize = os.stat(filename).st_size jsStr = self.ugoira_data[:-1] + r',"zipSize":' + str(zipSize) + r'}' with zipfile.ZipFile(zipTarget, mode="a") as z: z.writestr("animation.json", jsStr)
def login(self): oauth_response = None need_relogin = True if self._refresh_token is not None: PixivHelper.GetLogger().info("Login to OAuth using refresh token.") oauth_response = requests.post(self._url, self._get_values_for_refresh(), headers=self._get_default_headers(), proxies=self._proxies, verify=self._validate_ssl) if oauth_response.status_code == 200: need_relogin = False else: PixivHelper.GetLogger().info( "OAuth Refresh Token invalid, Relogin needed.") if need_relogin: oauth_response = self.login_with_username_and_password() PixivHelper.GetLogger().debug("{0}: {1}".format( oauth_response.status_code, oauth_response.text)) if oauth_response.status_code == 200: info = json.loads(oauth_response.text) self._refresh_token = info["response"]["refresh_token"] self._access_token = info["response"]["access_token"] elif oauth_response.status_code == 400: info = oauth_response.text try: info = json.loads(info)["errors"]["system"]["message"] except (ValueError, KeyError): pass PixivHelper.print_and_log('error', info) raise PixivException("Failed to login using OAuth", PixivException.OAUTH_LOGIN_ISSUE) return oauth_response
def WriteUgoiraData(self, filename): info = None try: # Issue #421 ensure subdir exists. PixivHelper.makeSubdirs(filename) info = codecs.open(filename, 'wb', encoding='utf-8') except IOError: info = codecs.open(str(self.imageId) + ".js", 'wb', encoding='utf-8') PixivHelper.GetLogger().exception( "Error when saving image info: %s, file is saved to: %d.js", filename, self.imageId) info.write(str(self.ugoira_data)) info.close()
def getBrowser(config=None, cookieJar=None): global defaultCookieJar global defaultConfig if config != None: defaultConfig = config if cookieJar != None: defaultCookieJar = cookieJar if defaultCookieJar == None: PixivHelper.GetLogger().info( "No default cookie jar available, creating... ") defaultCookieJar = cookielib.LWPCookieJar() browser = Browser(factory=mechanize.RobustFactory()) configureBrowser(browser, defaultConfig) configureCookie(browser, defaultCookieJar) return browser
def getBrowser(config=None, cookieJar=None): global defaultCookieJar global defaultConfig global _browser if _browser is None: if config is not None: defaultConfig = config if cookieJar is not None: defaultCookieJar = cookieJar if defaultCookieJar is None: PixivHelper.GetLogger().info("No default cookie jar available, creating... ") defaultCookieJar = cookielib.LWPCookieJar() _browser = PixivBrowser(defaultConfig, defaultCookieJar) return _browser
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None): artist = None response = None if tags is not None: tags = PixivHelper.encode_tags(tags) if self._isWhitecube: limit = 50 if bookmark: (url, response) = self.getMemberBookmarkWhiteCube( member_id, page, limit, tags) else: offset = (page - 1) * limit url = 'https://www.pixiv.net/rpc/whitecube/index.php?mode=user_new_unified&id={0}&offset_illusts={1}&offset_novels={2}&limit={3}&tt={4}'.format( member_id, offset, 0, limit, self._whitecubeToken) if tags is not None: url = url + '&tag={0}'.format(tags) elif self._config.r18mode: url = url + '&tag=R-18' PixivHelper.print_and_log('info', 'Member Url: ' + url) if url is not None: response = self.open(url).read() PixivHelper.GetLogger().debug(response) artist = PixivModelWhiteCube.PixivArtist( member_id, response, False) self.getMemberInfoWhitecube(member_id, artist, bookmark) else: if bookmark: member_url = 'https://www.pixiv.net/bookmark.php?id=' + str( member_id) + '&p=' + str(page) else: member_url = 'https://www.pixiv.net/member_illust.php?id=' + str( member_id) + '&p=' + str(page) if tags is not None: member_url = member_url + "&tag=" + tags elif self._config.r18mode and not bookmark: member_url = member_url + '&tag=R-18' PixivHelper.print_and_log('info', 'R-18 Mode only.') PixivHelper.print_and_log('info', 'Member Url: ' + member_url) response = self.getPixivPage(member_url) artist = PixivModel.PixivArtist(mid=member_id, page=response) return (artist, response)
def loginHttp(self, username, password): """ Log in to Pixiv, return 0 if success """ try: PixivHelper.printAndLog('info', 'Log in using form.') req = self._makeRequest(PixivConstant.PIXIV_URL + PixivConstant.PIXIV_LOGIN_URL) self.open(req) self.select_form(predicate=lambda f: f.attrs.get('action', None) == '/login.php') self['pixiv_id'] = username self['pass'] = password if self._config.keepSignedIn: self.find_control('skip').items[0].selected = True response = self.submit() return self.processLoginResult(response) except: PixivHelper.printAndLog('error', 'Error at pixiv_login():' + str(sys.exc_info())) PixivHelper.GetLogger().exception('Error at pixiv_login(): ' + str(sys.exc_info())) raise
def ParseBookmarkDetails(self, page): if page == None: raise PixivException('No page given', errorCode = PixivException.NO_PAGE_GIVEN) try: countUl = page.findAll('ul', attrs={'class':'count-list'}) if countUl is not None and len(countUl) > 0: countA = countUl[0].findAll('a') if countA is not None and len(countA) > 0: for a in countA: if "bookmark-count" in a["class"]: self.bookmark_count = int(a.text) elif "image-response-count" in a["class"]: self.image_response_count = int(a.text) return ## no bookmark count self.bookmark_count = 0 self.image_response_count = 0 except: PixivHelper.GetLogger().exception("Cannot parse bookmark count for: " + str(self.imageId))