def prepare():
# Log in
username = __config__.username
if username == '':
username = input('Username ? ').rstrip("\r")
password = __config__.password
if password == '':
password = getpass.getpass('Password ? ')
result = False
if len(__config__.cookie) > 0:
result = __br__.loginUsingCookie(__config__.cookie)
if not result:
result = __br__.login(username, password)
__br__._oauth_manager = PixivOAuth(
username,
password,
proxies=None,
refresh_token=__config__.refresh_token,
validate_ssl=__config__.enableSSLVerification)
__br__._oauth_manager.login()
return result
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
info = None
if artist.reference_image_id > 0:
url = "https://www.pixiv.net/rpc/get_work.php?id={0}".format(artist.reference_image_id)
PixivHelper.GetLogger().debug("using webrpc: %s", url)
info = self._get_from_cache(url)
if info is None:
request = urllib2.Request(url)
infoStr = self.open_with_retry(request).read()
info = json.loads(infoStr)
self._put_to_cache(url, info)
else:
PixivHelper.print_and_log('info', 'Using OAuth to retrieve member info for: {0}'.format(member_id))
if self._username is None or self._username is None or len(self._username) < 0 or len(self._password) < 0:
raise PixivException("Empty Username or Password, please remove the cookie value and relogin, or add username/password to config.ini.")
if self._oauth_manager is None:
proxy = None
if _config.useProxy:
proxy = self._config.proxy
self._oauth_manager = PixivOAuth(self._username, self._password, proxies=proxy, refresh_token=self._config.refresh_token)
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id)
info = self._get_from_cache(url)
if info is None:
PixivHelper.GetLogger().debug("Getting member information: %s", member_id)
login_response = self._oauth_manager.login()
if login_response.status_code == 200:
login_response = json.loads(oauth_response.text)
self._config.refresh_token = info["response"]["refresh_token"]
self._config.writeConfig(path=self._config.configFileLocation)
response = self._oauth_manager.get_user_info(member_id)
info = json.loads(response.text)
self._put_to_cache(url, info)
PixivHelper.GetLogger().debug("reply: %s", response.text)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self._get_from_cache(url_ajax)
if info_ajax is None:
info_ajax_str = self.open_with_retry(url_ajax).read()
info_ajax = json.loads(info_ajax_str)
self._put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib2.HTTPError, error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if payload.has_key("message"):
msg = payload["message"]
elif payload.has_key("error") and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage)
else:
raise PixivException(msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)
class PixivBrowser(mechanize.Browser):
_config = None
_isWhitecube = False
_whitecubeToken = ""
_cache = dict()
_myId = 0
_username = None
_password = None
_oauth_manager = None
def _put_to_cache(self, key, item, expiration=3600):
expiry = time.time() + expiration
self._cache[key] = (item, expiry)
def _get_from_cache(self, key):
if key in self._cache.keys():
(item, expiry) = self._cache[key]
if expiry - time.time() > 0:
return item
else:
del item
self._cache.pop(key)
return None
def __init__(self, config, cookie_jar):
# fix #218
try:
mechanize.Browser.__init__(self, factory=mechanize.RobustFactory())
except BaseException:
PixivHelper.GetLogger().info("Using default factory (mechanize 3.x ?)")
mechanize.Browser.__init__(self)
self._configureBrowser(config)
self._configureCookie(cookie_jar)
def clear_history(self):
mechanize.Browser.clear_history(self)
return
def back(self):
mechanize.Browser.back(self)
return
def _configureBrowser(self, config):
if config is None:
PixivHelper.GetLogger().info("No config given")
return
global defaultConfig
if defaultConfig is None:
defaultConfig = config
self._config = config
if config.useProxy:
if config.proxyAddress.startswith('socks'):
parseResult = urlparse.urlparse(config.proxyAddress)
assert parseResult.scheme and parseResult.hostname and parseResult.port
socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4
socks.setdefaultproxy(socksType, parseResult.hostname, parseResult.port)
socks.wrapmodule(urllib)
socks.wrapmodule(urllib2)
socks.wrapmodule(httplib)
PixivHelper.GetLogger().info("Using SOCKS Proxy: %s", config.proxyAddress)
else:
self.set_proxies(config.proxy)
PixivHelper.GetLogger().info("Using Proxy: %s", config.proxyAddress)
# self.set_handle_equiv(True)
# self.set_handle_gzip(True)
self.set_handle_redirect(True)
self.set_handle_referer(True)
self.set_handle_robots(False)
self.set_debug_http(config.debugHttp)
if config.debugHttp:
PixivHelper.GetLogger().info('Debug HTTP enabled.')
# self.visit_response
self.addheaders = [('User-agent', config.useragent)]
# force utf-8, fix issue #184
self.addheaders = [('Accept-Charset', 'utf-8')]
socket.setdefaulttimeout(config.timeout)
def _configureCookie(self, cookie_jar):
if cookie_jar is not None:
self.set_cookiejar(cookie_jar)
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = cookie_jar
def addCookie(self, cookie):
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = cookielib.LWPCookieJar()
defaultCookieJar.set_cookie(cookie)
def open_with_retry(self, url, data=None,
timeout=mechanize._sockettimeout._GLOBAL_DEFAULT_TIMEOUT,
retry=0):
retry_count = 0
if retry == 0 and self._config is not None:
retry = self._config.retry
while True:
try:
return self.open(url, data, timeout)
except Exception as ex:
if isinstance(ex, urllib2.HTTPError):
raise
if retry_count < retry:
for t in range(1, self._config.retryWait):
print(t, end=' ')
time.sleep(1)
print('')
retry_count = retry_count + 1
else:
raise PixivException("Failed to get page: {0}, please check your internet connection/firewall/antivirus.".format(ex.message), errorCode=PixivException.SERVER_ERROR)
def getPixivPage(self, url, referer="https://www.pixiv.net", returnParsed=True):
''' get page from pixiv and return as parsed BeautifulSoup object or response object.
throw PixivException as server error
'''
url = self.fixUrl(url)
retry_count = 0
while True:
req = urllib2.Request(url)
req.add_header('Referer', referer)
try:
page = self.open_with_retry(req)
if returnParsed:
parsedPage = BeautifulSoup(page.read())
return parsedPage
else:
return page
except Exception as ex:
if isinstance(ex, urllib2.HTTPError):
if ex.code in [403, 404, 503]:
return BeautifulSoup(ex.read())
if retry_count < self._config.retry:
for t in range(1, self._config.retryWait):
print(t, end=' ')
time.sleep(1)
print('')
retry_count = retry_count + 1
else:
raise PixivException("Failed to get page: " + ex.message, errorCode=PixivException.SERVER_ERROR)
def fixUrl(self, url, useHttps=True):
# url = str(url)
if not url.startswith("http"):
if not url.startswith("/"):
url = "/" + url
if useHttps:
return "https://www.pixiv.net" + url
else:
return "http://www.pixiv.net" + url
return url
def _loadCookie(self, cookie_value):
""" Load cookie to the Browser instance """
ck = cookielib.Cookie(version=0, name='PHPSESSID', value=cookie_value, port=None,
port_specified=False, domain='pixiv.net', domain_specified=False,
domain_initial_dot=False, path='/', path_specified=True,
secure=False, expires=None, discard=True, comment=None,
comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
self.addCookie(ck)
## cookies = cookie_value.split(";")
## for cookie in cookies:
## temp = cookie.split("=")
## name = temp[0].strip()
## value= temp[1] if len(temp) > 1 else ""
## ck = cookielib.Cookie(version=0, name=name, value=value, port=None,
## port_specified=False, domain='pixiv.net', domain_specified=False,
## domain_initial_dot=False, path='/', path_specified=True,
## secure=False, expires=None, discard=True, comment=None,
## comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
## self.addCookie(ck)
def _getInitConfig(self, page):
init_config = page.find('input', attrs={'id': 'init-config'})
js_init_config = json.loads(init_config['value'])
return js_init_config
def loginUsingCookie(self, login_cookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if login_cookie is None or len(login_cookie) == 0:
login_cookie = self._config.cookie
if len(login_cookie) > 0:
PixivHelper.print_and_log('info', 'Trying to log in with saved cookie')
self._loadCookie(login_cookie)
res = self.open_with_retry('https://www.pixiv.net/mypage.php')
resData = res.read()
parsed = BeautifulSoup(resData)
if "logout.php" in resData:
PixivHelper.print_and_log('info', 'Login successful.')
PixivHelper.GetLogger().info('Logged in using cookie')
self.getMyId(parsed)
return True
else:
PixivHelper.GetLogger().info('Failed to log in using cookie')
PixivHelper.print_and_log('info', 'Cookie already expired/invalid.')
return False
def login(self, username, password):
try:
PixivHelper.print_and_log('info', 'Logging in...')
url = "https://accounts.pixiv.net/login"
page = self.open_with_retry(url)
# get the post key
parsed = BeautifulSoup(page)
js_init_config = self._getInitConfig(parsed)
data = {}
data['pixiv_id'] = username
data['password'] = password
data['captcha'] = ''
data['g_recaptcha_response'] = ''
data['return_to'] = 'https://www.pixiv.net'
data['lang'] = 'en'
data['post_key'] = js_init_config["pixivAccount.postKey"]
data['source'] = "accounts"
data['ref'] = ''
request = urllib2.Request("https://accounts.pixiv.net/api/login?lang=en", urllib.urlencode(data))
response = self.open_with_retry(request)
return self.processLoginResult(response, username, password)
except BaseException:
PixivHelper.print_and_log('error', 'Error at login(): {0}'.format(sys.exc_info()))
raise
def processLoginResult(self, response, username, password):
PixivHelper.GetLogger().info('Logging in, return url: %s', response.geturl())
# check the returned json
js = response.read()
PixivHelper.GetLogger().info(str(js))
result = json.loads(js)
# Fix Issue #181
if result["body"] is not None and result["body"].has_key("success"):
# cookie.value = self._ua_handlers['_cookies']
# PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value))
# self._config.cookie = cookie.value
# self._config.writeConfig(path=self._config.configFileLocation)
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(path=self._config.configFileLocation)
break
# check whitecube
page = self.open_with_retry(result["body"]["success"]["return_to"])
parsed = BeautifulSoup(page)
self.getMyId(parsed)
# store the username and password in memory for oAuth login
self._config.username = username
self._config.password = password
return True
else:
if result["body"] is not None and result["body"].has_key("validation_errors"):
PixivHelper.print_and_log('info', "Server reply: " + str(result["body"]["validation_errors"]))
else:
PixivHelper.print_and_log('info', 'Unknown login issue, please use cookie login method.')
return False
def getMyId(self, parsed):
''' Assume from main page '''
# pixiv.user.id = "189816";
temp = re.findall(r"pixiv.user.id = \"(\d+)\";", unicode(parsed))
if temp is not None:
self._myId = int(temp[0])
PixivHelper.print_and_log('info', 'My User Id: {0}.'.format(self._myId))
else:
PixivHelper.print_and_log('info', 'Unable to get User Id')
def parseLoginError(self, res):
page = BeautifulSoup(res.read())
r = page.findAll('span', attrs={'class': 'error'})
return r
def getImagePage(self, image_id, parent=None, from_bookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: %s", image_id)
url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(image_id)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("globalInitData") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivModelWhiteCube.PixivImage(image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(image_id)
meta_response = self.open_with_retry(ugoira_meta_url).read()
image.ParseUgoira(meta_response)
if parent is None:
if from_bookmark:
image.originalArtist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.originalArtist.artistId, image.originalArtist)
else:
image.artist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
except:
PixivHelper.GetLogger().error("Respose data: \r\n %s", response)
raise
return (image, response)
def handleDebugMediumPage(self, response, imageId):
if self._config.enableDump:
if self._config.dumpMediumPage:
dump_filename = "Medium Page for Image Id {0}.html".format(imageId)
PixivHelper.dumpHtml(dump_filename, response)
PixivHelper.print_and_log('info', 'Dumping html to: {0}'.format(dump_filename))
if self._config.debugHttp:
PixivHelper.safePrint(u"reply: {0}".format(PixivHelper.toUnicode(response)))
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
info = None
if artist.reference_image_id > 0:
url = "https://www.pixiv.net/rpc/get_work.php?id={0}".format(artist.reference_image_id)
PixivHelper.GetLogger().debug("using webrpc: %s", url)
info = self._get_from_cache(url)
if info is None:
request = urllib2.Request(url)
infoStr = self.open_with_retry(request).read()
info = json.loads(infoStr)
self._put_to_cache(url, info)
else:
PixivHelper.print_and_log('info', 'Using OAuth to retrieve member info for: {0}'.format(member_id))
if self._username is None or self._username is None or len(self._username) < 0 or len(self._password) < 0:
raise PixivException("Empty Username or Password, please remove the cookie value and relogin, or add username/password to config.ini.")
if self._oauth_manager is None:
proxy = None
if _config.useProxy:
proxy = self._config.proxy
self._oauth_manager = PixivOAuth(self._username, self._password, proxies=proxy, refresh_token=self._config.refresh_token)
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id)
info = self._get_from_cache(url)
if info is None:
PixivHelper.GetLogger().debug("Getting member information: %s", member_id)
login_response = self._oauth_manager.login()
if login_response.status_code == 200:
login_response = json.loads(oauth_response.text)
self._config.refresh_token = info["response"]["refresh_token"]
self._config.writeConfig(path=self._config.configFileLocation)
response = self._oauth_manager.get_user_info(member_id)
info = json.loads(response.text)
self._put_to_cache(url, info)
PixivHelper.GetLogger().debug("reply: %s", response.text)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self._get_from_cache(url_ajax)
if info_ajax is None:
info_ajax_str = self.open_with_retry(url_ajax).read()
info_ajax = json.loads(info_ajax_str)
self._put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib2.HTTPError, error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if payload.has_key("message"):
msg = payload["message"]
elif payload.has_key("error") and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage)
else:
raise PixivException(msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)
class PixivBrowser(mechanize.Browser):
_config = None
_isWhitecube = False
_whitecubeToken = ""
_cache = dict()
_max_cache = 10000 # keep 1000 item in memory
_myId = 0
_isPremium = False
_username = None
_password = None
_oauth_manager = None
_locale = ""
def _put_to_cache(self, key, item, expiration=3600):
expiry = time.time() + expiration
self._cache[key] = (item, expiry)
# check oldest item
oldest_expiry = expiry
oldest_item = key
if len(self._cache) > self._max_cache:
for key2 in self._cache:
curr_expiry = self._cache[key2][1]
if curr_expiry < oldest_expiry:
oldest_item = key2
oldest_expiry = curr_expiry
del self._cache[oldest_item]
def _get_from_cache(self, key, sliding_window=3600):
if key in self._cache.keys():
(item, expiry) = self._cache.pop(key)
if expiry - time.time() > 0:
self._cache[key] = (item, expiry + sliding_window)
return item
# expired data
del item
return None
def __init__(self, config, cookie_jar):
# fix #218 not applicable after upgrading to mechanize 4.x
mechanize.Browser.__init__(self)
self._configureBrowser(config)
self._configureCookie(cookie_jar)
def clear_history(self):
mechanize.Browser.clear_history(self)
return
def back(self):
mechanize.Browser.back(self)
return
def _configureBrowser(self, config):
if config is None:
PixivHelper.get_logger().info("No config given")
return
global defaultConfig
if defaultConfig is None:
defaultConfig = config
self._config = config
if config.useProxy:
if config.proxyAddress.startswith('socks'):
parseResult = urllib.parse.urlparse(config.proxyAddress)
assert parseResult.scheme and parseResult.hostname and parseResult.port
socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4
PixivHelper.get_logger().info(f"Using SOCKS5 Proxy= {parseResult.hostname}:{parseResult.port}")
# https://stackoverflow.com/a/14512227
socks.setdefaultproxy(socksType, parseResult.hostname, parseResult.port)
socket.socket = socks.socksocket
else:
self.set_proxies(config.proxy)
PixivHelper.get_logger().info("Using Proxy: %s", config.proxyAddress)
# self.set_handle_equiv(True)
# self.set_handle_gzip(True)
self.set_handle_redirect(True)
self.set_handle_referer(True)
self.set_handle_robots(False)
self.set_debug_http(config.debugHttp)
if config.debugHttp:
PixivHelper.get_logger().info('Debug HTTP enabled.')
# self.visit_response
self.addheaders = [('User-agent', config.useragent)]
# force utf-8, fix issue #184
self.addheaders = [('Accept-Charset', 'utf-8')]
socket.setdefaulttimeout(config.timeout)
if not self._config.enableSSLVerification:
import ssl
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
# Legacy Python that doesn't verify HTTPS certificates by default
pass
else:
# Handle target environment that doesn't support HTTPS verification
ssl._create_default_https_context = _create_unverified_https_context
def _configureCookie(self, cookie_jar):
if cookie_jar is not None:
self.set_cookiejar(cookie_jar)
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = cookie_jar
def addCookie(self, cookie):
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = http.cookiejar.LWPCookieJar()
defaultCookieJar.set_cookie(cookie)
def clearCookie(self):
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = http.cookiejar.LWPCookieJar()
defaultCookieJar.clear()
def open_with_retry(self, url, data=None, timeout=60, retry=0):
''' Return response object with retry.'''
retry_count = 0
if retry == 0 and self._config is not None:
retry = self._config.retry
while True:
try:
return self.open(url, data, timeout)
except urllib.error.HTTPError:
raise
except BaseException:
exc_value = sys.exc_info()[1]
if retry_count < retry:
print(exc_value, end=' ')
for t in range(1, self._config.retryWait):
print(t, end=' ')
time.sleep(1)
print('')
retry_count = retry_count + 1
else:
temp = url
if isinstance(url, urllib.request.Request):
temp = url.full_url
PixivHelper.print_and_log('error', 'Error at open_with_retry(): {0}'.format(str(sys.exc_info())))
raise PixivException("Failed to get page: {0}, please check your internet connection/firewall/antivirus."
.format(temp), errorCode=PixivException.SERVER_ERROR)
def getPixivPage(self, url, referer="https://www.pixiv.net", returnParsed=True, enable_cache=True):
''' get page from pixiv and return as parsed BeautifulSoup object or response object.
throw PixivException as server error
'''
url = self.fixUrl(url)
while True:
req = mechanize.Request(url)
req.add_header('Referer', referer)
read_page = self._get_from_cache(url)
if read_page is None:
try:
temp = self.open_with_retry(req)
read_page = temp.read()
read_page = read_page.decode('utf8')
if enable_cache:
self._put_to_cache(url, read_page)
temp.close()
except urllib.error.HTTPError as ex:
if ex.code in [403, 404, 503]:
read_page = ex.read()
raise PixivException("Failed to get page: {0} => {1}".format(
url, ex), errorCode=PixivException.SERVER_ERROR)
else:
PixivHelper.print_and_log(
'error', 'Error at getPixivPage(): {0}'.format(str(sys.exc_info())))
raise PixivException("Failed to get page: {0}".format(
url), errorCode=PixivException.SERVER_ERROR)
if returnParsed:
parsedPage = BeautifulSoup(read_page, features="html5lib")
return parsedPage
return read_page
def fixUrl(self, url, useHttps=True):
# url = str(url)
if not url.startswith("http"):
if not url.startswith("/"):
url = "/" + url
if useHttps:
return "https://www.pixiv.net" + url
else:
return "http://www.pixiv.net" + url
return url
def _loadCookie(self, cookie_value):
""" Load cookie to the Browser instance """
ck = http.cookiejar.Cookie(version=0, name='PHPSESSID', value=cookie_value, port=None,
port_specified=False, domain='pixiv.net', domain_specified=False,
domain_initial_dot=False, path='/', path_specified=True,
secure=False, expires=None, discard=True, comment=None,
comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
self.addCookie(ck)
# cookies = cookie_value.split(";")
# for cookie in cookies:
# temp = cookie.split("=")
# name = temp[0].strip()
# value= temp[1] if len(temp) > 1 else ""
# ck = cookielib.Cookie(version=0, name=name, value=value, port=None,
# port_specified=False, domain='pixiv.net', domain_specified=False,
# domain_initial_dot=False, path='/', path_specified=True,
# secure=False, expires=None, discard=True, comment=None,
# comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
# self.addCookie(ck)
def _getInitConfig(self, page):
init_config = page.find('input', attrs={'id': 'init-config'})
js_init_config = json.loads(init_config['value'])
return js_init_config
def loginUsingCookie(self, login_cookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if login_cookie is None or len(login_cookie) == 0:
login_cookie = self._config.cookie
if len(login_cookie) > 0:
PixivHelper.print_and_log('info', 'Trying to log in with saved cookie')
self.clearCookie()
self._loadCookie(login_cookie)
res = self.open_with_retry('https://www.pixiv.net/')
parsed = BeautifulSoup(res, features="html5lib").decode('utf-8')
PixivHelper.get_logger().info('Logging in, return url: %s', res.geturl())
res.close()
result = False
if "logout.php" in str(parsed):
result = True
if "pixiv.user.loggedIn = true" in str(parsed):
result = True
if result:
PixivHelper.print_and_log('info', 'Login successful.')
PixivHelper.get_logger().info('Logged in using cookie')
self.getMyId(parsed)
temp_locale = str(res.geturl()).replace('https://www.pixiv.net/', '').replace('/', '')
if len(temp_locale) > 0:
self._locale = '/' + temp_locale
PixivHelper.get_logger().info('Locale = %s', self._locale)
else:
PixivHelper.get_logger().info('Failed to log in using cookie')
PixivHelper.print_and_log('info', 'Cookie already expired/invalid.')
del parsed
return result
def login(self, username, password):
parsed = None
try:
PixivHelper.print_and_log('info', 'Logging in...')
url = "https://accounts.pixiv.net/login"
# get the post key
res = self.open_with_retry(url)
parsed = BeautifulSoup(res, features="html5lib")
post_key = parsed.find('input', attrs={'name': 'post_key'})
# js_init_config = self._getInitConfig(parsed)
res.close()
data = {}
data['pixiv_id'] = username
data['password'] = password
# data['captcha'] = ''
# data['g_recaptcha_response'] = ''
data['return_to'] = 'https://www.pixiv.net'
data['lang'] = 'en'
data['post_key'] = post_key['value']
data['source'] = "accounts"
data['ref'] = ''
request = mechanize.Request("https://accounts.pixiv.net/api/login?lang=en", data, method='POST')
response = self.open_with_retry(request)
result = self.processLoginResult(response, username, password)
response.close()
return result
except BaseException:
traceback.print_exc()
PixivHelper.print_and_log('error', 'Error at login(): {0}'.format(sys.exc_info()))
PixivHelper.dump_html("login_error.html", str(parsed))
raise
finally:
if parsed is not None:
parsed.decompose()
del parsed
def processLoginResult(self, response, username, password):
PixivHelper.get_logger().info('Logging in, return url: %s', response.geturl())
# check the returned json
js = response.read()
PixivHelper.get_logger().info(str(js))
result = json.loads(js)
# Fix Issue #181
if result["body"] is not None and "success" in result["body"]:
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.print_and_log(
'info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(
path=self._config.configFileLocation)
break
# check whitecube
res = self.open_with_retry(result["body"]["success"]["return_to"])
parsed = BeautifulSoup(res, features="html5lib").decode('utf-8')
self.getMyId(parsed)
res.close()
# store the username and password in memory for oAuth login
self._config.username = username
self._config.password = password
del parsed
return True
else:
if result["body"] is not None and "validation_errors" in result["body"]:
PixivHelper.print_and_log(
'info', "Server reply: " + str(result["body"]["validation_errors"]))
if str(result["body"]["validation_errors"]).find("reCAPTCHA") > 0:
print(
"Please follow the method described in https://github.com/Nandaka/PixivUtil2/issues/505")
else:
PixivHelper.print_and_log(
'info', 'Unknown login issue, please use cookie login method.')
return False
def getMyId(self, parsed):
''' Assume from main page '''
# pixiv.user.id = "189816";
temp = re.findall(r"pixiv.user.id = \"(\d+)\";", parsed)
if temp is not None:
self._myId = int(temp[0])
PixivHelper.print_and_log(
'info', 'My User Id: {0}.'.format(self._myId))
else:
PixivHelper.print_and_log('info', 'Unable to get User Id')
self._isPremium = False
temp = re.findall(r"pixiv.user.premium = (\w+);", parsed)
if temp is not None:
self._isPremium = True if temp[0] == "true" else False
PixivHelper.print_and_log(
'info', 'Premium User: {0}.'.format(self._isPremium))
def parseLoginError(self, res):
page = BeautifulSoup(res, features="html5lib")
r = page.findAll('span', attrs={'class': 'error'})
page.decompose()
del page
return r
def getImagePage(self, image_id, parent=None, from_bookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.get_logger().debug("Getting image page: %s", image_id)
# https://www.pixiv.net/en/artworks/76656661
url = "https://www.pixiv.net{1}/artworks/{0}".format(image_id, self._locale)
response = self.getPixivPage(url, returnParsed=False, enable_cache=False)
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("meta-preload-data") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivImage(image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(image_id)
res = self.open_with_retry(ugoira_meta_url)
meta_response = res.read()
image.ParseUgoira(meta_response)
res.close()
if parent is None:
if from_bookmark:
image.originalArtist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.originalArtist.artistId, image.originalArtist)
else:
image.artist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
except BaseException:
PixivHelper.get_logger().error("Respose data: \r\n %s", response)
raise
return (image, response)
def handleDebugMediumPage(self, response, imageId):
if self._config.enableDump:
if self._config.dumpMediumPage:
dump_filename = "Medium Page for Image Id {0}.html".format(imageId)
PixivHelper.dump_html(dump_filename, response)
PixivHelper.print_and_log('info', 'Dumping html to: {0}'.format(dump_filename))
if self._config.debugHttp:
PixivHelper.safePrint(u"reply: {0}".format(response))
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
info = None
if int(artist.reference_image_id) > 0:
url = "https://www.pixiv.net/rpc/get_work.php?id={0}".format(artist.reference_image_id)
PixivHelper.get_logger().debug("using webrpc: %s", url)
info = self._get_from_cache(url)
if info is None:
request = mechanize.Request(url)
res = self.open_with_retry(request)
infoStr = res.read()
res.close()
info = json.loads(infoStr)
self._put_to_cache(url, info)
else:
PixivHelper.print_and_log('info', 'Using OAuth to retrieve member info for: {0}'.format(member_id))
if self._username is None or self._password is None or len(self._username) < 0 or len(self._password) < 0:
raise PixivException("Empty Username or Password, remove cookie value and relogin, or add username/password to config.ini.")
if self._oauth_manager is None:
proxy = None
if self._config.useProxy:
proxy = self._config.proxy
self._oauth_manager = PixivOAuth(self._username,
self._password,
proxies=proxy,
refresh_token=self._config.refresh_token,
validate_ssl=self._config.enableSSLVerification)
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(
member_id)
info = self._get_from_cache(url)
if info is None:
PixivHelper.get_logger().debug("Getting member information: %s", member_id)
login_response = self._oauth_manager.login()
if login_response.status_code == 200:
info = json.loads(login_response.text)
self._config.refresh_token = info["response"]["refresh_token"]
self._config.writeConfig(
path=self._config.configFileLocation)
response = self._oauth_manager.get_user_info(member_id)
info = json.loads(response.text)
self._put_to_cache(url, info)
PixivHelper.get_logger().debug("reply: %s", response.text)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self._get_from_cache(url_ajax)
if info_ajax is None:
res = self.open_with_retry(url_ajax)
info_ajax_str = res.read()
res.close()
info_ajax = json.loads(info_ajax_str)
self._put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib.error.HTTPError as error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.get_logger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if "message" in payload:
msg = payload["message"]
elif "error" in payload and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage)
else:
raise PixivException(msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is None:
tags = ''
limit = 48
offset = (page - 1) * limit
need_to_slice = False
if bookmark:
# https://www.pixiv.net/ajax/user/1039353/illusts/bookmarks?tag=&offset=0&limit=24&rest=show
url = 'https://www.pixiv.net/ajax/user/{0}/illusts/bookmarks?tag={1}&offset={2}&limit={3}&rest=show'
url = url.format(member_id, tags, offset, limit)
else:
# https://www.pixiv.net/ajax/user/1813972/illusts/tag?tag=Fate%2FGrandOrder?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1813972/manga/tag?tag=%E3%83%A1%E3%82%A4%E3%82%AD%E3%83%B3%E3%82%B0?offset=0&limit=24
# https://www.pixiv.net/ajax/user/5238/illustmanga/tag?tag=R-18&offset=0&limit=48
# https://www.pixiv.net/ajax/user/1813972/profile/all
url = None
if len(tags) > 0:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'
url = url.format(member_id, tags, offset, limit)
elif self._config.r18mode:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'
url = url.format(member_id, 'R-18', offset, limit)
else:
url = 'https://www.pixiv.net/ajax/user/{0}/profile/all'.format(member_id)
need_to_slice = True
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
# cache the response
response = self._get_from_cache(url)
if response is None:
try:
res = self.open_with_retry(url)
response = res.read()
res.close()
except urllib.error.HTTPError as ex:
if ex.code == 404:
response = ex.read()
self._put_to_cache(url, response)
PixivHelper.get_logger().debug(response)
artist = PixivArtist(member_id, response, False, offset, limit)
artist.reference_image_id = artist.imageList[0] if len(
artist.imageList) > 0 else 0
self.getMemberInfoWhitecube(member_id, artist, bookmark)
if artist.haveImages and need_to_slice:
artist.imageList = artist.imageList[offset:offset + limit]
return (artist, response)
def getSearchTagPage(self, tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1,
include_bookmark_data=False):
response_page = None
result = None
url = ''
if member_id is not None:
# from member id search by tags
(artist, response_page) = self.getMemberPage(
member_id, current_page, False, tags)
# convert to PixivTags
result = PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generate_search_tag_url(tags, current_page,
title_caption,
wild_card,
oldest_first,
start_date,
end_date,
member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for {0}'.format(url))
response_page = self.getPixivPage(url, returnParsed=False)
self.handleDebugTagSearchPage(response_page, url)
result = None
if member_id is not None:
result = PixivTags()
parse_search_page = BeautifulSoup(response_page, features="html5lib")
result.parseMemberTags(parse_search_page, member_id, tags)
parse_search_page.decompose()
del parse_search_page
else:
try:
result = PixivTags()
result.parseTags(response_page, tags, current_page)
# parse additional information
if include_bookmark_data:
idx = 0
print("Retrieving bookmark information...", end=' ')
for image in result.itemList:
idx = idx + 1
print("\r", end=' ')
print("Retrieving bookmark information... [{0}] of [{1}]".format(
idx, len(result.itemList)), end=' ')
img_url = "https://www.pixiv.net/ajax/illust/{0}".format(
image.imageId)
response_page = self._get_from_cache(img_url)
if response_page is None:
try:
res = self.open_with_retry(img_url)
response_page = res.read()
res.close()
except urllib.error.HTTPError as ex:
if ex.code == 404:
response_page = ex.read()
self._put_to_cache(img_url, response_page)
image_info_js = json.loads(response_page)
image.bookmarkCount = int(
image_info_js["body"]["bookmarkCount"])
image.imageResponse = int(
image_info_js["body"]["responseCount"])
print("")
except BaseException:
PixivHelper.dump_html("Dump for SearchTags " + tags + ".html", response_page)
raise
return (result, response_page)
def handleDebugTagSearchPage(self, response, url):
if self._config.enableDump:
if self._config.dumpTagSearchPage:
dump_filename = "TagSearch Page for {0}.html".format(url)
PixivHelper.dump_html(dump_filename, response)
PixivHelper.print_and_log(
'info', 'Dumping html to: {0}'.format(dump_filename))
if self._config.debugHttp:
PixivHelper.safePrint(u"reply: {0}".format(
PixivHelper.toUnicode(response)))
def fanboxGetSupportedUsers(self):
''' get all supported users from the list from https://fanbox.pixiv.net/api/plan.listSupporting'''
url = 'https://fanbox.pixiv.net/api/plan.listSupporting'
PixivHelper.print_and_log('info', f'Getting supported artists from {url}')
referer = "https://www.pixiv.net/fanbox/support/creators"
req = mechanize.Request(url)
req.add_header('Accept', 'application/json, text/plain, */*')
req.add_header('Referer', referer)
req.add_header('Origin', 'https://www.pixiv.net')
req.add_header('User-Agent', self._config.useragent)
res = self.open_with_retry(req)
# read the json response
response = res.read()
res.close()
result = Fanbox(response)
return result
def fanboxGetPostsFromArtist(self, artist_id, next_url=""):
''' get all posts from the supported user from https://fanbox.pixiv.net/api/post.listCreator?userId=1305019&limit=10 '''
# Issue #641
if next_url is None or next_url == "":
url = f"https://fanbox.pixiv.net/api/post.listCreator?userId={artist_id}&limit=10"
elif next_url.startswith("https://"):
url = next_url
else:
url = "https://www.pixiv.net" + next_url
# Fix #494
PixivHelper.print_and_log('info', 'Getting posts from ' + url)
referer = f"https://www.pixiv.net/fanbox/creator/{artist_id}"
req = mechanize.Request(url)
req.add_header('Accept', 'application/json, text/plain, */*')
req.add_header('Referer', referer)
req.add_header('Origin', 'https://www.pixiv.net')
req.add_header('User-Agent', self._config.useragent)
res = self.open_with_retry(req)
response = res.read()
PixivHelper.get_logger().debug(response.decode('utf8'))
res.close()
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
result = FanboxArtist(artist_id, response, tzInfo=_tzInfo)
pixivArtist = PixivArtist(artist_id)
self.getMemberInfoWhitecube(artist_id, pixivArtist)
result.artistName = pixivArtist.artistName
result.artistToken = pixivArtist.artistToken
for post in result.posts:
# https://fanbox.pixiv.net/api/post.info?postId=279561
# https://www.pixiv.net/fanbox/creator/104409/post/279561
p_url = f"https://fanbox.pixiv.net/api/post.info?postId={post.imageId}"
p_referer = f"https://www.pixiv.net/fanbox/creator/{artist_id}/post/{post.imageId}"
PixivHelper.get_logger().debug('Getting post detail from %s', p_url)
p_req = mechanize.Request(p_url)
p_req.add_header('Accept', 'application/json, text/plain, */*')
p_req.add_header('Referer', p_referer)
p_req.add_header('Origin', 'https://www.pixiv.net')
p_req.add_header('User-Agent', self._config.useragent)
p_res = self.open_with_retry(p_req)
p_response = p_res.read()
PixivHelper.get_logger().debug(p_response.decode('utf8'))
p_res.close()
js = demjson.decode(p_response)
post.parsePost(js["body"])
return result
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
info = None
if artist.reference_image_id > 0:
url = "https://www.pixiv.net/rpc/get_work.php?id={0}".format(artist.reference_image_id)
PixivHelper.GetLogger().debug("using webrpc: %s", url)
info = self._get_from_cache(url)
if info is None:
request = urllib2.Request(url)
infoStr = self.open_with_retry(request).read()
info = json.loads(infoStr)
self._put_to_cache(url, info)
else:
PixivHelper.print_and_log('info', 'Using OAuth to retrieve member info for: {0}'.format(member_id))
if self._username is None or self._username is None or len(self._username) < 0 or len(self._password) < 0:
raise PixivException("Empty Username or Password, please remove the cookie value and relogin, or add username/password to config.ini.")
if self._oauth_manager is None:
proxy = None
if self._config.useProxy:
proxy = self._config.proxy
self._oauth_manager = PixivOAuth(self._username, self._password, proxies=proxy, refresh_token=self._config.refresh_token)
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id)
info = self._get_from_cache(url)
if info is None:
PixivHelper.GetLogger().debug("Getting member information: %s", member_id)
login_response = self._oauth_manager.login()
if login_response.status_code == 200:
info = json.loads(login_response.text)
self._config.refresh_token = info["response"]["refresh_token"]
self._config.writeConfig(path=self._config.configFileLocation)
response = self._oauth_manager.get_user_info(member_id)
info = json.loads(response.text)
self._put_to_cache(url, info)
PixivHelper.GetLogger().debug("reply: %s", response.text)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self._get_from_cache(url_ajax)
if info_ajax is None:
info_ajax_str = self.open_with_retry(url_ajax).read()
info_ajax = json.loads(info_ajax_str)
self._put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib2.HTTPError, error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if payload.has_key("message"):
msg = payload["message"]
elif payload.has_key("error") and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage)
else:
raise PixivException(msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)
class PixivBrowser(mechanize.Browser):
_config = None
_isWhitecube = False
_whitecubeToken = ""
_cache = dict()
_myId = 0
_username = None
_password = None
_oauth_manager = None
def _put_to_cache(self, key, item, expiration=3600):
expiry = time.time() + expiration
self._cache[key] = (item, expiry)
def _get_from_cache(self, key):
if key in self._cache.keys():
(item, expiry) = self._cache[key]
if expiry - time.time() > 0:
return item
else:
del item
self._cache.pop(key)
return None
def __init__(self, config, cookie_jar):
# fix #218
try:
mechanize.Browser.__init__(self, factory=mechanize.RobustFactory())
except BaseException:
PixivHelper.GetLogger().info("Using default factory (mechanize 3.x ?)")
mechanize.Browser.__init__(self)
self._configureBrowser(config)
self._configureCookie(cookie_jar)
def clear_history(self):
mechanize.Browser.clear_history(self)
return
def back(self):
mechanize.Browser.back(self)
return
def _configureBrowser(self, config):
if config is None:
PixivHelper.GetLogger().info("No config given")
return
global defaultConfig
if defaultConfig is None:
defaultConfig = config
self._config = config
if config.useProxy:
if config.proxyAddress.startswith('socks'):
parseResult = urlparse.urlparse(config.proxyAddress)
assert parseResult.scheme and parseResult.hostname and parseResult.port
socksType = socks.PROXY_TYPE_SOCKS5 if parseResult.scheme == 'socks5' else socks.PROXY_TYPE_SOCKS4
socks.setdefaultproxy(socksType, parseResult.hostname, parseResult.port)
socks.wrapmodule(urllib)
socks.wrapmodule(urllib2)
socks.wrapmodule(httplib)
PixivHelper.GetLogger().info("Using SOCKS Proxy: %s", config.proxyAddress)
else:
self.set_proxies(config.proxy)
PixivHelper.GetLogger().info("Using Proxy: %s", config.proxyAddress)
# self.set_handle_equiv(True)
# self.set_handle_gzip(True)
self.set_handle_redirect(True)
self.set_handle_referer(True)
self.set_handle_robots(False)
self.set_debug_http(config.debugHttp)
if config.debugHttp:
PixivHelper.GetLogger().info('Debug HTTP enabled.')
# self.visit_response
self.addheaders = [('User-agent', config.useragent)]
# force utf-8, fix issue #184
self.addheaders = [('Accept-Charset', 'utf-8')]
socket.setdefaulttimeout(config.timeout)
def _configureCookie(self, cookie_jar):
if cookie_jar is not None:
self.set_cookiejar(cookie_jar)
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = cookie_jar
def addCookie(self, cookie):
global defaultCookieJar
if defaultCookieJar is None:
defaultCookieJar = cookielib.LWPCookieJar()
defaultCookieJar.set_cookie(cookie)
def open_with_retry(self, url, data=None,
timeout=mechanize._sockettimeout._GLOBAL_DEFAULT_TIMEOUT,
retry=0):
retry_count = 0
if retry == 0 and self._config is not None:
retry = self._config.retry
while True:
try:
return self.open(url, data, timeout)
except urllib2.HTTPError:
raise
except Exception as ex:
if retry_count < retry:
for t in range(1, self._config.retryWait):
print(t, end=' ')
time.sleep(1)
print('')
retry_count = retry_count + 1
else:
raise PixivException("Failed to get page: {0}, please check your internet connection/firewall/antivirus.".format(ex.message), errorCode=PixivException.SERVER_ERROR)
def getPixivPage(self, url, referer="https://www.pixiv.net", returnParsed=True):
''' get page from pixiv and return as parsed BeautifulSoup object or response object.
throw PixivException as server error
'''
url = self.fixUrl(url)
retry_count = 0
while True:
req = urllib2.Request(url)
req.add_header('Referer', referer)
try:
page = self.open_with_retry(req)
if returnParsed:
parsedPage = BeautifulSoup(page.read())
return parsedPage
else:
return page
except urllib2.HTTPError as ex:
if ex.code in [403, 404, 503]:
return BeautifulSoup(ex.read())
except Exception as ex:
if retry_count < self._config.retry:
for t in range(1, self._config.retryWait):
print(t, end=' ')
time.sleep(1)
print('')
retry_count = retry_count + 1
else:
raise PixivException("Failed to get page: " + ex.message, errorCode=PixivException.SERVER_ERROR)
def fixUrl(self, url, useHttps=True):
# url = str(url)
if not url.startswith("http"):
if not url.startswith("/"):
url = "/" + url
if useHttps:
return "https://www.pixiv.net" + url
else:
return "http://www.pixiv.net" + url
return url
def _loadCookie(self, cookie_value):
""" Load cookie to the Browser instance """
ck = cookielib.Cookie(version=0, name='PHPSESSID', value=cookie_value, port=None,
port_specified=False, domain='pixiv.net', domain_specified=False,
domain_initial_dot=False, path='/', path_specified=True,
secure=False, expires=None, discard=True, comment=None,
comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
self.addCookie(ck)
# cookies = cookie_value.split(";")
# for cookie in cookies:
# temp = cookie.split("=")
# name = temp[0].strip()
# value= temp[1] if len(temp) > 1 else ""
# ck = cookielib.Cookie(version=0, name=name, value=value, port=None,
# port_specified=False, domain='pixiv.net', domain_specified=False,
# domain_initial_dot=False, path='/', path_specified=True,
# secure=False, expires=None, discard=True, comment=None,
# comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
# self.addCookie(ck)
def _getInitConfig(self, page):
init_config = page.find('input', attrs={'id': 'init-config'})
js_init_config = json.loads(init_config['value'])
return js_init_config
def loginUsingCookie(self, login_cookie=None):
""" Log in to Pixiv using saved cookie, return True if success """
if login_cookie is None or len(login_cookie) == 0:
login_cookie = self._config.cookie
if len(login_cookie) > 0:
PixivHelper.print_and_log('info', 'Trying to log in with saved cookie')
self._loadCookie(login_cookie)
res = self.open_with_retry('https://www.pixiv.net/mypage.php')
resData = res.read()
parsed = BeautifulSoup(resData)
if "logout.php" in resData:
PixivHelper.print_and_log('info', 'Login successful.')
PixivHelper.GetLogger().info('Logged in using cookie')
self.getMyId(parsed)
return True
else:
PixivHelper.GetLogger().info('Failed to log in using cookie')
PixivHelper.print_and_log('info', 'Cookie already expired/invalid.')
return False
def login(self, username, password):
try:
PixivHelper.print_and_log('info', 'Logging in...')
url = "https://accounts.pixiv.net/login"
page = self.open_with_retry(url)
# get the post key
parsed = BeautifulSoup(page)
js_init_config = self._getInitConfig(parsed)
data = {}
data['pixiv_id'] = username
data['password'] = password
data['captcha'] = ''
data['g_recaptcha_response'] = ''
data['return_to'] = 'https://www.pixiv.net'
data['lang'] = 'en'
data['post_key'] = js_init_config["pixivAccount.postKey"]
data['source'] = "accounts"
data['ref'] = ''
request = urllib2.Request("https://accounts.pixiv.net/api/login?lang=en", urllib.urlencode(data))
response = self.open_with_retry(request)
return self.processLoginResult(response, username, password)
except BaseException:
PixivHelper.print_and_log('error', 'Error at login(): {0}'.format(sys.exc_info()))
raise
def processLoginResult(self, response, username, password):
PixivHelper.GetLogger().info('Logging in, return url: %s', response.geturl())
# check the returned json
js = response.read()
PixivHelper.GetLogger().info(str(js))
result = json.loads(js)
# Fix Issue #181
if result["body"] is not None and result["body"].has_key("success"):
# cookie.value = self._ua_handlers['_cookies']
# PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value))
# self._config.cookie = cookie.value
# self._config.writeConfig(path=self._config.configFileLocation)
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.print_and_log('info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(path=self._config.configFileLocation)
break
# check whitecube
page = self.open_with_retry(result["body"]["success"]["return_to"])
parsed = BeautifulSoup(page)
self.getMyId(parsed)
# store the username and password in memory for oAuth login
self._config.username = username
self._config.password = password
return True
else:
if result["body"] is not None and result["body"].has_key("validation_errors"):
PixivHelper.print_and_log('info', "Server reply: " + str(result["body"]["validation_errors"]))
else:
PixivHelper.print_and_log('info', 'Unknown login issue, please use cookie login method.')
return False
def getMyId(self, parsed):
''' Assume from main page '''
# pixiv.user.id = "189816";
temp = re.findall(r"pixiv.user.id = \"(\d+)\";", unicode(parsed))
if temp is not None:
self._myId = int(temp[0])
PixivHelper.print_and_log('info', 'My User Id: {0}.'.format(self._myId))
else:
PixivHelper.print_and_log('info', 'Unable to get User Id')
def parseLoginError(self, res):
page = BeautifulSoup(res.read())
r = page.findAll('span', attrs={'class': 'error'})
return r
def getImagePage(self, image_id, parent=None, from_bookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: %s", image_id)
url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(image_id)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("globalInitData") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivModelWhiteCube.PixivImage(image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(image_id)
meta_response = self.open_with_retry(ugoira_meta_url).read()
image.ParseUgoira(meta_response)
if parent is None:
if from_bookmark:
image.originalArtist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.originalArtist.artistId, image.originalArtist)
else:
image.artist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
except:
PixivHelper.GetLogger().error("Respose data: \r\n %s", response)
raise
return (image, response)
def handleDebugMediumPage(self, response, imageId):
if self._config.enableDump:
if self._config.dumpMediumPage:
dump_filename = "Medium Page for Image Id {0}.html".format(imageId)
PixivHelper.dumpHtml(dump_filename, response)
PixivHelper.print_and_log('info', 'Dumping html to: {0}'.format(dump_filename))
if self._config.debugHttp:
PixivHelper.safePrint(u"reply: {0}".format(PixivHelper.toUnicode(response)))
def getMemberInfoWhitecube(self, member_id, artist, bookmark=False):
''' get artist information using Ajax and AppAPI '''
try:
info = None
if artist.reference_image_id > 0:
url = "https://www.pixiv.net/rpc/get_work.php?id={0}".format(artist.reference_image_id)
PixivHelper.GetLogger().debug("using webrpc: %s", url)
info = self._get_from_cache(url)
if info is None:
request = urllib2.Request(url)
infoStr = self.open_with_retry(request).read()
info = json.loads(infoStr)
self._put_to_cache(url, info)
else:
PixivHelper.print_and_log('info', 'Using OAuth to retrieve member info for: {0}'.format(member_id))
if self._username is None or self._username is None or len(self._username) < 0 or len(self._password) < 0:
raise PixivException("Empty Username or Password, please remove the cookie value and relogin, or add username/password to config.ini.")
if self._oauth_manager is None:
proxy = None
if self._config.useProxy:
proxy = self._config.proxy
self._oauth_manager = PixivOAuth(self._username, self._password, proxies=proxy, refresh_token=self._config.refresh_token)
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(member_id)
info = self._get_from_cache(url)
if info is None:
PixivHelper.GetLogger().debug("Getting member information: %s", member_id)
login_response = self._oauth_manager.login()
if login_response.status_code == 200:
info = json.loads(login_response.text)
self._config.refresh_token = info["response"]["refresh_token"]
self._config.writeConfig(path=self._config.configFileLocation)
response = self._oauth_manager.get_user_info(member_id)
info = json.loads(response.text)
self._put_to_cache(url, info)
PixivHelper.GetLogger().debug("reply: %s", response.text)
artist.ParseInfo(info, False, bookmark=bookmark)
# will throw HTTPError if user is suspended/not logged in.
url_ajax = 'https://www.pixiv.net/ajax/user/{0}'.format(member_id)
info_ajax = self._get_from_cache(url_ajax)
if info_ajax is None:
info_ajax_str = self.open_with_retry(url_ajax).read()
info_ajax = json.loads(info_ajax_str)
self._put_to_cache(url_ajax, info_ajax)
# 2nd pass to get the background
artist.ParseBackground(info_ajax)
return artist
except urllib2.HTTPError, error:
errorCode = error.getcode()
errorMessage = error.get_data()
PixivHelper.GetLogger().error("Error data: \r\n %s", errorMessage)
payload = demjson.decode(errorMessage)
# Issue #432
if payload.has_key("message"):
msg = payload["message"]
elif payload.has_key("error") and payload["error"] is not None:
msgs = list()
msgs.append(payload["error"]["user_message"])
msgs.append(payload["error"]["message"])
msgs.append(payload["error"]["reason"])
msg = ",".join(msgs)
if errorCode == 401:
raise PixivException(msg, errorCode=PixivException.NOT_LOGGED_IN, htmlPage=errorMessage)
elif errorCode == 403:
raise PixivException(msg, errorCode=PixivException.USER_ID_SUSPENDED, htmlPage=errorMessage)
else:
raise PixivException(msg, errorCode=PixivException.OTHER_MEMBER_ERROR, htmlPage=errorMessage)