def loginUsingCookie(self, login_cookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if login_cookie is None or len(login_cookie) == 0:
login_cookie = self._config.cookie
if len(login_cookie) > 0:
PixivHelper.print_and_log('info', 'Trying to log in with saved cookie')
self.clearCookie()
self._loadCookie(login_cookie)
res = self.open_with_retry('https://www.pixiv.net/')
resData = res.read()
parsed = BeautifulSoup(resData)
PixivHelper.GetLogger().info('Logging in, return url: %s', res.geturl())
if "logout.php" in resData:
PixivHelper.print_and_log('info', 'Login successful.')
PixivHelper.GetLogger().info('Logged in using cookie')
self.getMyId(parsed)
temp_locale = str(res.geturl()).replace('https://www.pixiv.net/', '').replace('/', '')
if len(temp_locale) > 0:
self._locale = '/' + temp_locale
PixivHelper.GetLogger().info('Locale = %s', self._locale)
return True
else:
PixivHelper.GetLogger().info('Failed to log in using cookie')
PixivHelper.print_and_log('info', 'Cookie already expired/invalid.')
return False
def processLoginResult(self, response, username, password):
PixivHelper.GetLogger().info('Logging in, return url: %s', response.geturl())
# check the returned json
js = response.read()
PixivHelper.GetLogger().info(str(js))
result = json.loads(js)
# Fix Issue #181
if result["body"] is not None and result["body"].has_key("success"):
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(path=self._config.configFileLocation)
break
# check whitecube
page = self.open_with_retry(result["body"]["success"]["return_to"])
parsed = BeautifulSoup(page)
self.getMyId(parsed)
# store the username and password in memory for oAuth login
self._config.username = username
self._config.password = password
return True
else:
if result["body"] is not None and result["body"].has_key("validation_errors"):
PixivHelper.print_and_log('info', "Server reply: " + str(result["body"]["validation_errors"]))
if str(result["body"]["validation_errors"]).find("reCAPTCHA") > 0:
print("Please follow the method described in https://github.com/Nandaka/PixivUtil2/issues/505")
else:
PixivHelper.print_and_log('info', 'Unknown login issue, please use cookie login method.')
return False
def getImagePage(self, imageId, parent=None, fromBookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: {0}".format(imageId))
if self._isWhitecube:
url = "https://www.pixiv.net/rpc/whitecube/index.php?mode=work_details_modal_whitecube&id={0}&tt={1}".format(imageId, self._whitecubeToken)
response = self.open(url).read()
PixivHelper.GetLogger().debug(response);
image = PixivModelWhiteCube.PixivImage(imageId,
response,
parent,
fromBookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat)
# overwrite artist info
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
else:
url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(imageId)
response = self.open(url).read()
parsed = BeautifulSoup(response)
image = PixivModel.PixivImage(imageId,
parsed,
parent,
fromBookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat)
if image.imageMode == "ugoira_view" or image.imageMode == "bigNew":
image.ParseImages(parsed)
parsed.decompose()
return (image, response)
def loginUsingCookie(self, login_cookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if login_cookie is None or len(login_cookie) == 0:
login_cookie = self._config.cookie
if len(login_cookie) > 0:
PixivHelper.print_and_log('info',
'Trying to log in with saved cookie')
self._loadCookie(login_cookie)
res = self.open('https://www.pixiv.net/mypage.php')
resData = res.read()
parsed = BeautifulSoup(resData)
self.detectWhiteCube(parsed, res.geturl())
if "logout.php" in resData:
PixivHelper.print_and_log('info', 'Login successful.')
PixivHelper.GetLogger().info('Logged in using cookie')
self.getMyId(parsed)
return True
else:
PixivHelper.GetLogger().info('Failed to log in using cookie')
PixivHelper.print_and_log('info',
'Cookie already expired/invalid.')
return False
def processLoginResult(self, response):
PixivHelper.GetLogger().info('Logging in, return url: ' + response.geturl())
# check the returned json
js = response.read()
PixivHelper.GetLogger().info(str(js))
result = json.loads(js)
# Fix Issue #181
if result["body"] is not None and result["body"].has_key("success"):
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.printAndLog('info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(path=self._config.configFileLocation)
break
# check whitecube
page = self.open(result["body"]["success"]["return_to"])
parsed = BeautifulSoup(page)
self.detectWhiteCube(parsed, page.geturl())
return True
else:
if result["body"] is not None and result["body"].has_key("validation_errors"):
PixivHelper.printAndLog('info', "Server reply: " + str(result["body"]["validation_errors"]))
else:
PixivHelper.printAndLog('info', 'Unknown login issue, please use cookie login method.')
return False
def getImagePage(self,
image_id,
parent=None,
from_bookmark=False,
bookmark_count=-1,
image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: %s", image_id)
url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(
image_id)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("globalInitData") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivModelWhiteCube.PixivImage(
image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(
image_id)
meta_response = self.open_with_retry(
ugoira_meta_url).read()
image.ParseUgoira(meta_response)
if parent is None:
if from_bookmark:
image.originalArtist.reference_image_id = image_id
self.getMemberInfoWhitecube(
image.originalArtist.artistId,
image.originalArtist)
else:
image.artist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.artist.artistId,
image.artist)
except:
PixivHelper.GetLogger().error("Respose data: \r\n %s", response)
raise
return (image, response)
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename) :
raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 1
try:
for line in reader:
originalLine = line
##PixivHelper.safePrint("Processing: " + line)
if line.startswith('#') or len(line) < 1:
continue
if len(line.strip()) == 0:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
lineNo = lineNo + 1
originalLine = ""
except UnicodeDecodeError:
PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'.format(originalLine, lineNo))
except:
PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(
member_id)
info = self.get_from_cache(url)
if info is None:
PixivHelper.GetLogger().debug("Getting member information: %s",
member_id)
infoStr = self.open(url).read()
info = json.loads(infoStr)
self.put_to_cache(url, info)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self.get_from_cache(url_ajax)
if info_ajax is None:
info_ajax_str = self.open(url_ajax).read()
info_ajax = json.loads(info_ajax_str)
self.put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib2.HTTPError, error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if payload.has_key("message"):
msg = payload["message"]
elif payload.has_key("error") and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg,
errorCode=PixivException.NOT_LOGGED_IN,
htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(
msg,
errorCode=PixivException.USER_ID_SUSPENDED,
htmlPage=errorMessage)
else:
raise PixivException(
msg,
errorCode=PixivException.OTHER_MEMBER_ERROR,
htmlPage=errorMessage)
def ParseWorksData(self, page):
temp = page.find(attrs={'class': 'meta'}).findAll('li')
#07/22/2011 03:09|512×600|RETAS STUDIO
#07/26/2011 00:30|Manga 39P|ComicStudio 鉛筆 つけペン
#1/05/2011 07:09|723×1023|Photoshop SAI [ R-18 ]
#2013年3月16日 06:44 | 800×1130 | Photoshop ComicStudio | R-18
#2013年12月14日 19:00 855×1133 PhotoshopSAI
self.worksDate = PixivHelper.toUnicode(temp[0].string,
encoding=sys.stdin.encoding)
if self.dateFormat is not None and len(
self.dateFormat) > 0 and '%' in self.dateFormat:
# use the user defined format
try:
self.worksDateDateTime = datetime.strptime(
self.worksDate, self.dateFormat)
except ValueError as ve:
PixivHelper.GetLogger().exception(
'Error when parsing datetime: {0} for imageId {1} using date format {2}'
.format(self.worksDate, str(self.imageId),
str(self.dateFormat)), ve)
raise
else:
self.worksDate = self.worksDate.replace(u'/', u'-')
if self.worksDate.find('-') > -1:
try:
self.worksDateDateTime = datetime.strptime(
self.worksDate, u'%m-%d-%Y %H:%M')
except ValueError as ve:
PixivHelper.GetLogger().exception(
'Error when parsing datetime: {0} for imageId {1}'.
format(self.worksDate, self.imageId), ve)
self.worksDateDateTime = datetime.strptime(
self.worksDate.split(" ")[0], u'%Y-%m-%d')
else:
tempDate = self.worksDate.replace(u'年', '-').replace(
u'月', '-').replace(u'日', '')
self.worksDateDateTime = datetime.strptime(
tempDate, '%Y-%m-%d %H:%M')
self.worksResolution = unicode(temp[1].string).replace(u'×', u'x')
toolsTemp = page.find(attrs={
'class': 'meta'
}).find(attrs={'class': 'tools'})
if toolsTemp != None and len(toolsTemp) > 0:
tools = toolsTemp.findAll('li')
for tool in tools:
self.worksTools = self.worksTools + ' ' + unicode(tool.string)
self.worksTools = self.worksTools.strip()
class TestPixivModel_WhiteCube(unittest.TestCase):
currPath = unicode(os.path.abspath('.'))
PixivHelper.GetLogger()
def testParseLoginForm(self):
p = open('./test/pixiv-whitecube-main.html', 'r')
page = BeautifulSoup(p.read())
init_config = page.find('input', attrs={'id': 'init-config'})
js_init_config = json.loads(init_config['value'])
self.assertIsNotNone(js_init_config)
self.assertIsNotNone(js_init_config["pixiv.context.token"])
def testParseImage(self):
p = open('./test/work_details_modal_whitecube.json', 'r')
image = PixivImage(59521621, p.read())
self.assertIsNotNone(image)
image.PrintInfo()
self.assertEqual(image.imageMode, "big")
def testParseManga(self):
p = open('./test/work_details_modal_whitecube-manga.json', 'r')
image = PixivImage(59532028, p.read())
self.assertIsNotNone(image)
image.PrintInfo()
self.assertEqual(image.imageMode, "manga")
def WriteInfo(self, filename):
info = None
try:
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".txt",
'wb',
encoding='utf-8')
PixivHelper.GetLogger().exception(
"Error when saving image info: " + filename +
", file is saved to: " + str(self.imageId) + ".txt")
info.write("ArtistID = " + str(self.artist.artistId) + "\r\n")
info.write("ArtistName = " + self.artist.artistName + "\r\n")
info.write("ImageID = " + str(self.imageId) + "\r\n")
info.write("Title = " + self.imageTitle + "\r\n")
info.write("Caption = " + self.imageCaption + "\r\n")
info.write("Tags = " + ", ".join(self.imageTags) + "\r\n")
info.write("Image Mode = " + self.imageMode + "\r\n")
info.write("Pages = " + str(self.imageCount) + "\r\n")
info.write("Date = " + self.worksDate + "\r\n")
info.write("Resolution = " + self.worksResolution + "\r\n")
info.write("Tools = " + self.worksTools + "\r\n")
info.write("BookmarkCount= " + str(self.bookmark_count) + "\r\n")
info.write(
"Link = http://www.pixiv.net/member_illust.php?mode=medium&illust_id="
+ str(self.imageId) + "\r\n")
info.write("Ugoira Data= " + str(self.ugoira_data) + "\r\n")
info.close()
def WriteJSON(self, filename):
info = None
try:
info = codecs.open(filename, 'w', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".txt",
'w',
encoding='utf-8')
PixivHelper.GetLogger().exception(
"Error when saving image info: " + filename +
", file is saved to: " + str(self.imageId) + ".txt")
info.write("{" + "\r\n")
info.write("\t" + json.dumps("Artist ID") + ": " +
json.dumps(self.artist.artistId, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Artist Name") + ": " +
json.dumps(self.artist.artistName, ensure_ascii=False) +
"," + "\r\n")
info.write("\t" + json.dumps("Image ID") + ": " +
json.dumps(self.imageId, ensure_ascii=False) + "," + "\r\n")
info.write("\t" + json.dumps("Title") + ": " +
json.dumps(self.imageTitle, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Caption") + ": " +
json.dumps(self.imageCaption, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Tags") + ": " +
json.dumps(self.imageTags, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Image Mode") + ": " +
json.dumps(self.imageMode, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Pages") + ": " +
json.dumps(self.imageCount, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Date") + ": " +
json.dumps(self.worksDate, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Resolution") + ": " +
json.dumps(self.worksResolution, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Tools") + ": " +
json.dumps(self.worksTools, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("BookmarkCount") + ": " +
json.dumps(self.bookmark_count, ensure_ascii=False) + "," +
"\r\n")
info.write("\t" + json.dumps("Link") + ": " + json.dumps(
"http://www.pixiv.net/member_illust.php?mode=medium&illust_id=" +
str(self.imageId),
ensure_ascii=False) + "," + "\r\n")
info.write("\t" + json.dumps("Ugoira Data") + ": " +
json.dumps(self.ugoira_data, ensure_ascii=False) + "\r\n")
if len(self.descriptionUrlList) > 0:
info.write(
"\t" + json.dumps("Urls") + ": " +
json.dumps(self.descriptionUrlList, ensure_ascii=False) + "," +
"\r\n")
info.write("}")
info.close()
def processLoginResult(self, response):
PixivHelper.GetLogger().info('Logging in, return url: ' +
response.geturl())
## failed login will return to either of these page:
## http://www.pixiv.net/login.php
## https://www.secure.pixiv.net/login.php
if response.geturl().find('pixiv.net/login.php') == -1:
PixivHelper.printAndLog('info', 'Logged in')
## write back the new cookie value
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.printAndLog(
'info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(
path=self._config.configFileLocation)
break
return True
else:
errors = self.parseLoginError(response)
if len(errors) > 0:
for error in errors:
if error.string is not None:
PixivHelper.printAndLog(
'error', 'Server Reply: ' + error.string)
else:
PixivHelper.printAndLog('info', 'Wrong username or password.')
return False
def WriteInfo(self, filename):
info = None
try:
# Issue #421 ensure subdir exists.
PixivHelper.makeSubdirs(filename)
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".txt", 'wb', encoding='utf-8')
PixivHelper.GetLogger().exception("Error when saving image info: %s, file is saved to: %s.txt", filename, self.imageId)
info.write(u"ArtistID = {0}\r\n".format(self.parent.artistId))
info.write(u"ArtistName = {0}\r\n".format(self.parent.artistName))
info.write(u"ImageID = {0}\r\n".format(self.imageId))
info.write(u"Title = {0}\r\n".format(self.imageTitle))
info.write(u"Caption = {0}\r\n".format(self.body_text))
# info.write(u"Tags = " + ", ".join(self.imageTags) + "\r\n")
if self.is_restricted:
info.write(u"Image Mode = {0}, Restricted\r\n".format(self.type))
else:
info.write(u"Image Mode = {0}\r\n".format(self.type))
info.write(u"Pages = {0}\r\n".format(self.imageCount))
info.write(u"Date = {0}\r\n".format(self.worksDate))
# info.write(u"Resolution = " + self.worksResolution + "\r\n")
# info.write(u"Tools = " + self.worksTools + "\r\n")
info.write(u"Like Count = {0}\r\n".format(self.likeCount))
info.write(u"Link = https://www.pixiv.net/fanbox/creator/{0}/post/{1}\r\n".format(self.parent.artistId, self.imageId))
# info.write("Ugoira Data = " + str(self.ugoira_data) + "\r\n")
if len(self.embeddedFiles) > 0:
info.write("Urls =\r\n")
for link in self.embeddedFiles:
info.write(" - {0}\r\n".format(link))
info.close()
def _configureBrowser(self, config):
if config is None:
PixivHelper.GetLogger().info("No config given")
return
global defaultConfig
if defaultConfig is None:
defaultConfig = config
self._config = config
if config.useProxy:
if config.proxyAddress.startswith('socks'):
parseResult = urlparse.urlparse(config.proxyAddress)
assert parseResult.scheme and parseResult.hostname and parseResult.port
socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4
socks.setdefaultproxy(socksType, parseResult.hostname,
parseResult.port)
socks.wrapmodule(urllib)
socks.wrapmodule(urllib2)
socks.wrapmodule(httplib)
PixivHelper.GetLogger().info("Using SOCKS Proxy: %s",
config.proxyAddress)
else:
self.set_proxies(config.proxy)
PixivHelper.GetLogger().info("Using Proxy: %s",
config.proxyAddress)
# self.set_handle_equiv(True)
# self.set_handle_gzip(True)
self.set_handle_redirect(True)
self.set_handle_referer(True)
self.set_handle_robots(False)
self.set_debug_http(config.debugHttp)
if config.debugHttp:
PixivHelper.GetLogger().info('Debug HTTP enabled.')
# self.visit_response
self.addheaders = [('User-agent', config.useragent)]
# force utf-8, fix issue #184
self.addheaders = [('Accept-Charset', 'utf-8')]
socket.setdefaulttimeout(config.timeout)
def WriteUgoiraData(self, filename):
info = None
try:
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".js", 'wb', encoding='utf-8')
PixivHelper.GetLogger().exception("Error when saving image info: " + filename + ", file is saved to: " + str(self.imageId) + ".js")
info.write(str(self.ugoira_data))
info.close()
def login_with_username_and_password(self):
PixivHelper.GetLogger().info(
"Login to OAuth using username and password.")
oauth_response = requests.post(self._url,
self._get_values_for_login(),
headers=self._get_default_headers(),
proxies=self._proxies,
verify=self._validate_ssl)
return oauth_response
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is not None:
tags = PixivHelper.encode_tags(tags)
else:
tags = ''
limit = 48
offset = (page - 1) * limit
need_to_slice = False
if bookmark:
# https://www.pixiv.net/ajax/user/1039353/illusts/bookmarks?tag=&offset=0&limit=24&rest=show
url = 'https://www.pixiv.net/ajax/user/{0}/illusts/bookmarks?tag={1}&offset={2}&limit={3}&rest=show'.format(
member_id, tags, offset, limit)
else:
# https://www.pixiv.net/ajax/user/1813972/illusts/tag?tag=Fate%2FGrandOrder?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1813972/manga/tag?tag=%E3%83%A1%E3%82%A4%E3%82%AD%E3%83%B3%E3%82%B0?offset=0&limit=24
# https://www.pixiv.net/ajax/user/5238/illustmanga/tag?tag=R-18&offset=0&limit=48
# https://www.pixiv.net/ajax/user/1813972/profile/all
url = None
if len(tags) > 0:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format(
member_id, tags, offset, limit)
elif self._config.r18mode:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format(
member_id, 'R-18', offset, limit)
else:
url = 'https://www.pixiv.net/ajax/user/{0}/profile/all'.format(
member_id)
need_to_slice = True
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
# cache the response
response = self._get_from_cache(url)
if response is None:
try:
response = self.open_with_retry(url).read()
except urllib.error.HTTPError as ex:
if ex.code == 404:
response = ex.read()
self._put_to_cache(url, response)
PixivHelper.GetLogger().debug(response)
artist = PixivModelWhiteCube.PixivArtist(member_id, response,
False, offset, limit)
artist.reference_image_id = artist.imageList[0] if len(
artist.imageList) > 0 else 0
self.getMemberInfoWhitecube(member_id, artist, bookmark)
if artist.haveImages and need_to_slice:
artist.imageList = artist.imageList[offset:offset + limit]
return (artist, response)
def __init__(self, config, cookie_jar):
# fix #218
try:
mechanize.Browser.__init__(self, factory=mechanize.RobustFactory())
except BaseException:
PixivHelper.GetLogger().info("Using default factory (mechanize 3.x ?)")
mechanize.Browser.__init__(self)
self._configureBrowser(config)
self._configureCookie(cookie_jar)
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using AppAPI '''
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id)
if self._cache.has_key(url):
info = self._cache[url]
else:
PixivHelper.GetLogger().debug("Getting member information: {0}".format(member_id))
infoStr = self.open(url).read()
info = json.loads(infoStr)
self._cache[url] = info
artist.ParseInfo(info, False, bookmark=bookmark)
def loginUsingCookie(self, loginCookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if loginCookie is None or len(loginCookie) == 0:
loginCookie = self._config.cookie
if len(loginCookie) > 0:
PixivHelper.printAndLog('info', 'Trying to log with saved cookie')
self._loadCookie(loginCookie)
req = self._makeRequest('http://www.pixiv.net/mypage.php')
self.open(req)
res_url = self.response().geturl()
if res_url == 'http://www.pixiv.net/mypage.php':
PixivHelper.printAndLog('info', 'Login successfull.')
PixivHelper.GetLogger().info('Logged in using cookie')
return True
else:
PixivHelper.GetLogger().info('Failed to login using cookie, returned page: ' + res_url)
PixivHelper.printAndLog('info', 'Cookie already expired/invalid.')
return False
def configureBrowser(browser, config):
if config == None:
PixivHelper.GetLogger().info("No config given")
return
global defaultConfig
if defaultConfig == None:
defaultConfig = config
if config.useProxy:
if config.proxyAddress.startswith('socks'):
parseResult = urlparse.urlparse(config.proxyAddress)
assert parseResult.scheme and parseResult.hostname and parseResult.port
socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5,
parseResult.hostname, parseResult.port)
socks.wrapmodule(urllib)
socks.wrapmodule(urllib2)
socks.wrapmodule(httplib)
PixivHelper.GetLogger().info("Using SOCKS Proxy: " +
config.proxyAddress)
else:
browser.set_proxies(config.proxy)
PixivHelper.GetLogger().info("Using Proxy: " + config.proxyAddress)
browser.set_handle_equiv(True)
#browser.set_handle_gzip(True)
browser.set_handle_redirect(True)
browser.set_handle_referer(True)
browser.set_handle_robots(config.useRobots)
browser.set_debug_http(config.debugHttp)
if config.debugHttp:
PixivHelper.GetLogger().info('Debug HTTP enabled.')
browser.visit_response
browser.addheaders = [('User-agent', config.useragent)]
socket.setdefaulttimeout(config.timeout)
def CreateUgoira(self, filename):
if len(self.ugoira_data) == 0:
PixivHelper.GetLogger().exception("Missing ugoira animation info for image: " + str(self.imageId))
zipTarget = filename[:-4] + ".ugoira"
if os.path.exists(zipTarget):
os.remove(zipTarget)
shutil.copyfile(filename, zipTarget)
zipSize = os.stat(filename).st_size
jsStr = self.ugoira_data[:-1] + r',"zipSize":' + str(zipSize) + r'}'
with zipfile.ZipFile(zipTarget, mode="a") as z:
z.writestr("animation.json", jsStr)
def login(self):
oauth_response = None
need_relogin = True
if self._refresh_token is not None:
PixivHelper.GetLogger().info("Login to OAuth using refresh token.")
oauth_response = requests.post(self._url,
self._get_values_for_refresh(),
headers=self._get_default_headers(),
proxies=self._proxies,
verify=self._validate_ssl)
if oauth_response.status_code == 200:
need_relogin = False
else:
PixivHelper.GetLogger().info(
"OAuth Refresh Token invalid, Relogin needed.")
if need_relogin:
oauth_response = self.login_with_username_and_password()
PixivHelper.GetLogger().debug("{0}: {1}".format(
oauth_response.status_code, oauth_response.text))
if oauth_response.status_code == 200:
info = json.loads(oauth_response.text)
self._refresh_token = info["response"]["refresh_token"]
self._access_token = info["response"]["access_token"]
elif oauth_response.status_code == 400:
info = oauth_response.text
try:
info = json.loads(info)["errors"]["system"]["message"]
except (ValueError, KeyError):
pass
PixivHelper.print_and_log('error', info)
raise PixivException("Failed to login using OAuth",
PixivException.OAUTH_LOGIN_ISSUE)
return oauth_response
def WriteUgoiraData(self, filename):
info = None
try:
# Issue #421 ensure subdir exists.
PixivHelper.makeSubdirs(filename)
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".js",
'wb',
encoding='utf-8')
PixivHelper.GetLogger().exception(
"Error when saving image info: %s, file is saved to: %d.js",
filename, self.imageId)
info.write(str(self.ugoira_data))
info.close()
def getBrowser(config=None, cookieJar=None):
global defaultCookieJar
global defaultConfig
if config != None:
defaultConfig = config
if cookieJar != None:
defaultCookieJar = cookieJar
if defaultCookieJar == None:
PixivHelper.GetLogger().info(
"No default cookie jar available, creating... ")
defaultCookieJar = cookielib.LWPCookieJar()
browser = Browser(factory=mechanize.RobustFactory())
configureBrowser(browser, defaultConfig)
configureCookie(browser, defaultCookieJar)
return browser
def getBrowser(config=None, cookieJar=None):
global defaultCookieJar
global defaultConfig
global _browser
if _browser is None:
if config is not None:
defaultConfig = config
if cookieJar is not None:
defaultCookieJar = cookieJar
if defaultCookieJar is None:
PixivHelper.GetLogger().info("No default cookie jar available, creating... ")
defaultCookieJar = cookielib.LWPCookieJar()
_browser = PixivBrowser(defaultConfig, defaultCookieJar)
return _browser
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is not None:
tags = PixivHelper.encode_tags(tags)
if self._isWhitecube:
limit = 50
if bookmark:
(url, response) = self.getMemberBookmarkWhiteCube(
member_id, page, limit, tags)
else:
offset = (page - 1) * limit
url = 'https://www.pixiv.net/rpc/whitecube/index.php?mode=user_new_unified&id={0}&offset_illusts={1}&offset_novels={2}&limit={3}&tt={4}'.format(
member_id, offset, 0, limit, self._whitecubeToken)
if tags is not None:
url = url + '&tag={0}'.format(tags)
elif self._config.r18mode:
url = url + '&tag=R-18'
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
response = self.open(url).read()
PixivHelper.GetLogger().debug(response)
artist = PixivModelWhiteCube.PixivArtist(
member_id, response, False)
self.getMemberInfoWhitecube(member_id, artist, bookmark)
else:
if bookmark:
member_url = 'https://www.pixiv.net/bookmark.php?id=' + str(
member_id) + '&p=' + str(page)
else:
member_url = 'https://www.pixiv.net/member_illust.php?id=' + str(
member_id) + '&p=' + str(page)
if tags is not None:
member_url = member_url + "&tag=" + tags
elif self._config.r18mode and not bookmark:
member_url = member_url + '&tag=R-18'
PixivHelper.print_and_log('info', 'R-18 Mode only.')
PixivHelper.print_and_log('info', 'Member Url: ' + member_url)
response = self.getPixivPage(member_url)
artist = PixivModel.PixivArtist(mid=member_id, page=response)
return (artist, response)
def loginHttp(self, username, password):
""" Log in to Pixiv, return 0 if success """
try:
PixivHelper.printAndLog('info', 'Log in using form.')
req = self._makeRequest(PixivConstant.PIXIV_URL + PixivConstant.PIXIV_LOGIN_URL)
self.open(req)
self.select_form(predicate=lambda f: f.attrs.get('action', None) == '/login.php')
self['pixiv_id'] = username
self['pass'] = password
if self._config.keepSignedIn:
self.find_control('skip').items[0].selected = True
response = self.submit()
return self.processLoginResult(response)
except:
PixivHelper.printAndLog('error', 'Error at pixiv_login():' + str(sys.exc_info()))
PixivHelper.GetLogger().exception('Error at pixiv_login(): ' + str(sys.exc_info()))
raise
def ParseBookmarkDetails(self, page):
if page == None:
raise PixivException('No page given', errorCode = PixivException.NO_PAGE_GIVEN)
try:
countUl = page.findAll('ul', attrs={'class':'count-list'})
if countUl is not None and len(countUl) > 0:
countA = countUl[0].findAll('a')
if countA is not None and len(countA) > 0:
for a in countA:
if "bookmark-count" in a["class"]:
self.bookmark_count = int(a.text)
elif "image-response-count" in a["class"]:
self.image_response_count = int(a.text)
return
## no bookmark count
self.bookmark_count = 0
self.image_response_count = 0
except:
PixivHelper.GetLogger().exception("Cannot parse bookmark count for: " + str(self.imageId))
