def getImagePage(self, imageId, parent=None, fromBookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: {0}".format(imageId))
if self._isWhitecube:
url = "https://www.pixiv.net/rpc/whitecube/index.php?mode=work_details_modal_whitecube&id={0}&tt={1}".format(imageId, self._whitecubeToken)
response = self.open(url).read()
PixivHelper.GetLogger().debug(response);
image = PixivModelWhiteCube.PixivImage(imageId,
response,
parent,
fromBookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat)
# overwrite artist info
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
else:
url = "http://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(imageId)
response = self.open(url).read()
parsed = BeautifulSoup(response)
image = PixivModel.PixivImage(imageId,
parsed,
parent,
fromBookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat)
if image.imageMode == "ugoira_view" or image.imageMode == "bigNew":
image.ParseImages(parsed)
parsed.decompose()
return (image, response)
def createAvatarFilename(artistPage, targetDir):
filename = ''
image = PixivModel.PixivImage(parent=artistPage)
# Download avatar using custom name, refer issue #174
if len(_config.avatarNameFormat) > 0:
filenameFormat = _config.avatarNameFormat
filename = makeFilename(filenameFormat,
image,
tagsSeparator=_config.tagsSeparator,
tagsLimit=_config.tagsLimit,
fileUrl=artistPage.artistAvatar,
appendExtension=True)
filename = sanitizeFilename(filename, targetDir)
else:
# or as folder.jpg
filenameFormat = _config.filenameFormat
if filenameFormat.find(os.sep) == -1:
filenameFormat = os.sep + filenameFormat
filenameFormat = os.sep.join(filenameFormat.split(os.sep)[:-1])
filename = makeFilename(filenameFormat,
image,
tagsSeparator=_config.tagsSeparator,
tagsLimit=_config.tagsLimit,
fileUrl=artistPage.artistAvatar,
appendExtension=False)
filename = sanitizeFilename(filename + os.sep + 'folder.jpg',
targetDir)
return filename
def parseMemberTags(self, artist, memberId, query=""):
'''process artist result and return the image list'''
self.itemList = list()
self.memberId = memberId
self.query = query
self.haveImage = artist.haveImages
self.isLastPage = artist.isLastPage
for image in artist.imageList:
self.itemList.append(PixivModel.PixivTagsItem(int(image), 0, 0))
def CreateAvatarFilename(filenameFormat, tagsSeparator, tagsLimit, artistPage, targetDir):
filename = ''
if filenameFormat.find(os.sep) == -1:
filenameFormat = os.sep + filenameFormat
filenameFormat = os.sep.join(filenameFormat.split(os.sep)[:-1])
image = PixivModel.PixivImage(parent=artistPage)
filename = makeFilename(filenameFormat, image, tagsSeparator=tagsSeparator, tagsLimit=tagsLimit,
fileUrl=artistPage.artistAvatar, appendExtension=False)
filename = sanitizeFilename(filename + os.sep + 'folder.jpg', targetDir)
return filename
def main(user, max_times, private, download_type, works_type, tags_include,
tags_exclude):
'''
A simple CMD tool for bookmarks download and user's works download.
Supports database updating.
Download type: 0: Bookmarks | 1: User's works
'''
try:
tags_include = None if not tags_include else set(
tags_include.split(';'))
tags_exclude = None if not tags_exclude else set(
tags_exclude.split(';'))
except:
print('Value USER | MAX | DOWNLOAD_TYPE must be INT.')
exit(-1)
pcfg = PixivConfig.PixivConfig('config.json')
papi = PixivAPI.PixivAPI(logger=pcfg.get_logger('PixivAPI'))
if pcfg.cfg['pixiv']['refresh_token']:
login_result = papi.login(
refresh_token=pcfg.cfg['pixiv']['refresh_token'])
else:
import getpass
username = input('E-mail / Pixiv ID: ')
password = getpass.getpass()
login_result = papi.login(username, password)
if login_result['status_code'] != 0:
exit(-1)
logger = pcfg.get_logger('PixivCMD')
pdb = PixivModel.PixivDB(pcfg.database_uri)
pdl = PixivDownloader.PixivDownloader(
pcfg.pixiv_works_dir, logger=pcfg.get_logger('PixivDownloader'))
if download_type == 'bookmarks':
logger.info('Downloading all bookmarks...')
elif download_type == 'user':
logger.info('Downloading user\'s works...')
if not user:
user = papi.pixiv_user_id
pdl.all_works(download_type, papi, pdb.sessionmaker(), user, max_times,
works_type, tags_include, tags_exclude)
while True:
if pdl.dq.unfinished_tasks:
time.sleep(0.1)
else:
break
pdl.dq.join()
logger.info('Works download task for user %s done!' % papi.pixiv_user_id)
def getSearchTagPage(self, tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1):
response = None
result = None
url = ''
if member_id is not None:
# from member id search by tags
(artist, response) = self.getMemberPage(member_id, current_page, False, tags)
# convert to PixivTags
result = PixivModelWhiteCube.PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generateSearchTagUrl(tags, current_page,
title_caption,
wild_card,
oldest_first,
start_date,
end_date,
member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for ' + url)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugTagSearchPage(response, url)
parse_search_page = BeautifulSoup(response)
result = PixivModel.PixivTags()
if member_id is not None:
result.parseMemberTags(parse_search_page, member_id, tags)
else:
try:
result.parseTags(parse_search_page, tags)
except BaseException:
PixivHelper.dumpHtml("Dump for SearchTags " + tags + ".html", response)
raise
parse_search_page.decompose()
del parse_search_page
return (result, response)
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is not None:
tags = PixivHelper.encode_tags(tags)
if self._isWhitecube:
limit = 50
if bookmark:
(url, response) = self.getMemberBookmarkWhiteCube(
member_id, page, limit, tags)
else:
offset = (page - 1) * limit
url = 'https://www.pixiv.net/rpc/whitecube/index.php?mode=user_new_unified&id={0}&offset_illusts={1}&offset_novels={2}&limit={3}&tt={4}'.format(
member_id, offset, 0, limit, self._whitecubeToken)
if tags is not None:
url = url + '&tag={0}'.format(tags)
elif self._config.r18mode:
url = url + '&tag=R-18'
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
response = self.open(url).read()
PixivHelper.GetLogger().debug(response)
artist = PixivModelWhiteCube.PixivArtist(
member_id, response, False)
self.getMemberInfoWhitecube(member_id, artist, bookmark)
else:
if bookmark:
member_url = 'https://www.pixiv.net/bookmark.php?id=' + str(
member_id) + '&p=' + str(page)
else:
member_url = 'https://www.pixiv.net/member_illust.php?id=' + str(
member_id) + '&p=' + str(page)
if tags is not None:
member_url = member_url + "&tag=" + tags
elif self._config.r18mode and not bookmark:
member_url = member_url + '&tag=R-18'
PixivHelper.print_and_log('info', 'R-18 Mode only.')
PixivHelper.print_and_log('info', 'Member Url: ' + member_url)
response = self.getPixivPage(member_url)
artist = PixivModel.PixivArtist(mid=member_id, page=response)
return (artist, response)
def Artist2Tasks(self, artist, path=''):
#format: artist/type/
types = ['illust', 'manga'] #,'ugoira']
works_dir = path + str(artist.uid) + '_' + artist.content['name'] + '/'
if not os.path.exists(works_dir):
os.mkdir(works_dir)
for ctype in types:
if ctype in artist.content:
cdir = works_dir + str(ctype) + '/'
if not os.path.exists(cdir):
os.mkdir(cdir)
for wid in artist.content[ctype]:
work = PixivModel.Work(id=wid)
work.crawlData()
self.Work2Tasks(work=work, path=cdir)
def parseTags(self, page, query=""):
payload = json.loads(page)
self.query = query
# check error
if payload["error"]:
raise PixivException('Image Error: ' + payload["message"],
errorCode=PixivException.SERVER_ERROR)
# parse image information
parsed = BeautifulSoup(payload["body"]["html"])
self.itemList = list()
images = parsed.findAll("div",
attrs={"class": self.__re_imageItemClass})
for item in images:
thumbnail_container = item.find(
"div", attrs={"class": "thumbnail-container"})
image_details = thumbnail_container.find(
"a", attrs={"class": "_work-modal-target user-activity"})
image_id = image_details["data-work-id"]
# like count
status_container = thumbnail_container.find(
"div", attrs={"class": "status-container"})
bookmarkCount = status_container.text
imageResponse = 0
self.itemList.append(
PixivModel.PixivTagsItem(int(image_id), int(bookmarkCount),
int(imageResponse)))
if len(self.itemList) > 0:
self.haveImage = True
else:
self.haveImage = False
# search page info
self.availableImages = int(payload["body"]["total"])
if len(payload["body"]["next_url"]) > 0:
self.isLastPage = False
else:
self.isLastPage = True
return self.itemList
def parseTags(self, page, query="", curr_page=1):
payload = json.loads(page)
self.query = query
# check error
if payload["error"]:
raise PixivException('Image Error: ' + payload["message"],
errorCode=PixivException.SERVER_ERROR)
# parse images information
self.itemList = list()
for item in payload["body"]["illustManga"]["data"]:
if item["isAdContainer"]:
continue
image_id = item["id"]
# like count not available anymore, need to call separate request...
bookmarkCount = 0
imageResponse = 0
tag_item = PixivModel.PixivTagsItem(int(image_id),
int(bookmarkCount),
int(imageResponse))
self.itemList.append(tag_item)
self.haveImage = False
if len(self.itemList) > 0:
self.haveImage = True
# search page info
self.availableImages = int(payload["body"]["illustManga"]["total"])
# assuming there are only 47 image (1 is marked as ad)
if self.availableImages > 47 * curr_page:
self.isLastPage = False
else:
self.isLastPage = True
return self.itemList
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None, user_dir=''):
artist = None
response = None
if self._isWhitecube:
limit = 50
if bookmark:
PixivHelper.printAndLog('info', 'Getting Bookmark Url for page {0}...'.format(page))
# iterate to get next page url
start = 1
last_member_bookmark_next_url = None
while start <= page:
if start == 1:
url = 'https://www.pixiv.net/rpc/whitecube/index.php?mode=user_collection_unified&id={0}&bookmark_restrict={1}&limit={2}&is_profile_page={3}&is_first_request={4}&max_illust_bookmark_id={5}&max_novel_bookmark_id={6}&tt={7}'
url = url.format(member_id, 0, limit, 1, 1, 0, 0, self._whitecubeToken)
else:
url = last_member_bookmark_next_url
# PixivHelper.printAndLog('info', 'Member Bookmark Page {0} Url: {1}'.format(start, url))
if self._cache.has_key(url):
response = self._cache[url]
else:
response = self.open(url).read()
self._cache[url] = response
payload = json.loads(response)
last_member_bookmark_next_url = payload["body"]["next_url"]
if last_member_bookmark_next_url is None and start < page:
PixivHelper.printAndLog('info', 'No more images for {0} bookmarks'.format(member_id))
url = None
break
start = start + 1
PixivHelper.printAndLog('info', 'Member Bookmark Page {0} Url: {1}'.format(page, url))
else:
offset = (page - 1) * limit
url = 'https://www.pixiv.net/rpc/whitecube/index.php?mode=user_new_unified&id={0}&offset_illusts={1}&offset_novels={2}&limit={3}&tt={4}'.format(member_id, offset, 0, limit, self._whitecubeToken)
PixivHelper.printAndLog('info', 'Member Url: ' + url)
if url is not None:
response = self.open(url).read()
PixivHelper.GetLogger().debug(response);
artist = PixivModelWhiteCube.PixivArtist(member_id, response, False)
self.getMemberInfoWhitecube(member_id, artist, bookmark)
else:
if bookmark:
member_url = 'http://www.pixiv.net/bookmark.php?id=' + str(member_id) + '&p=' + str(page)
if tags is not None:
tags = encode_tags(tags)
member_url = member_url + "&tag=" + tags
else:
member_url = 'http://www.pixiv.net/member_illust.php?id=' + str(member_id) + '&p=' + str(page)
if self._config.r18mode and not bookmark:
member_url = member_url + '&tag=R-18'
PixivHelper.printAndLog('info', 'R-18 Mode only.')
PixivHelper.printAndLog('info', 'Member Url: ' + member_url)
response = self.getPixivPage(member_url)
artist = PixivModel.PixivArtist(mid=member_id, page=response)
return (artist, response)
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is not None:
tags = PixivHelper.encode_tags(tags)
else:
tags = ''
if True:
limit = 24
need_to_slice = False
if bookmark:
# (url, response) = self.getMemberBookmarkWhiteCube(member_id, page, limit, tags)
# https://www.pixiv.net/ajax/user/1039353/illusts/bookmarks?tag=&offset=0&limit=24&rest=show
url = 'https://www.pixiv.net/ajax/user/{0}/illusts/bookmarks?tag={1}&offset={2}&limit={3}'.format(
member_id, tags, offset, limit)
else:
# https://www.pixiv.net/ajax/user/1813972/illusts/tag/Fate%2FGrandOrder?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1813972/manga/tag/%E3%83%A1%E3%82%A4%E3%82%AD%E3%83%B3%E3%82%B0?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1113943/illustmanga/tag/%E6%A5%B5%E4%B8%8A%E3%81%AE%E4%B9%B3?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1813972/profile/all
offset = (page - 1) * limit
url = None
if len(tags) > 0:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag/{1}?offset={2}&limit={3}'.format(
member_id, tags, offset, limit)
elif self._config.r18mode:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag/{1}?offset={2}&limit={3}'.format(
member_id, 'R-18', offset, limit)
else:
url = 'https://www.pixiv.net/ajax/user/{0}/profile/all'.format(
member_id)
need_to_slice = True
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
# cache the response
response = self.get_from_cache(url)
if response is None:
response = self.open(url).read()
self.put_to_cache(url, response)
PixivHelper.GetLogger().debug(response)
artist = PixivModelWhiteCube.PixivArtist(
member_id, response, False, offset, limit)
self.getMemberInfoWhitecube(member_id, artist, bookmark)
if artist.haveImages and need_to_slice:
artist.imageList = artist.imageList[offset:offset + limit]
else:
if bookmark:
member_url = 'https://www.pixiv.net/bookmark.php?id=' + str(
member_id) + '&p=' + str(page)
else:
member_url = 'https://www.pixiv.net/member_illust.php?id=' + str(
member_id) + '&p=' + str(page)
if len(tags) > 0:
member_url = member_url + "&tag=" + tags
elif self._config.r18mode and not bookmark:
member_url = member_url + '&tag=R-18'
PixivHelper.print_and_log('info', 'R-18 Mode only.')
PixivHelper.print_and_log('info', 'Member Url: ' + member_url)
response = self.getPixivPage(member_url)
artist = PixivModel.PixivArtist(mid=member_id, page=response)
return (artist, response)
for ctype in types:
if ctype in artist.content:
cdir = works_dir + str(ctype) + '/'
if not os.path.exists(cdir):
os.mkdir(cdir)
for wid in artist.content[ctype]:
work = PixivModel.Work(id=wid)
work.crawlData()
self.Work2Tasks(work=work, path=cdir)
def download(self):
self.downloader.run()
if __name__ == '__main__':
#os.chdir('pixiv')
PixivModel.Login()
pc = PixivCrawler()
artist = PixivModel.Artist(170597)
artist.crawlData()
pc.Artist2Tasks(artist, 'pixiv/')
pc.download()
#work = PixivModel.Work(69001507)
#work.crawlData()
#pc.Work2Tasks(work, 'As109/illust/')
#while not pc.Q.empty():
#print(pc.Q.get())
def getSearchTagPage(self,
tags,
i,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1):
response = None
result = None
if self._isWhitecube:
if member_id is None:
# from search page:
# https://www.pixiv.net/rpc/whitecube/index.php?order=date&adult_mode=include&q=vocaloid&p=0&type=&mode=whitecube_search&s_mode=s_tag&scd=&size=&ratio=&like=&tools=&tt=4e2cdee233f1156231ee99da1e51a83c
url = "https://www.pixiv.net/rpc/whitecube/index.php?q={0}".format(
tags)
url = url + "&adult_mode={0}".format("include")
url = url + "&mode={0}".format("whitecube_search")
# date ordering
order = "date_d"
if oldest_first:
order = "date"
url = url + "&order={0}".format(order)
# search mode
s_mode = "s_tag_full"
if wild_card:
s_mode = "s_tag"
elif title_caption:
s_mode = "s_tc"
url = url + "&s_mode={0}".format(s_mode)
# start/end date
if start_date is not None:
url = url + "&scd={0}".format(start_date)
if end_date is not None:
url = url + "&ecd={0}".format(end_date)
url = url + "&p={0}".format(i)
url = url + "&start_page={0}".format(start_page)
url = url + "&tt={0}".format(self._whitecubeToken)
PixivHelper.printAndLog('info',
'Looping for {0} ...'.format(url))
response = self.open(url).read()
PixivHelper.GetLogger().debug(response)
result = PixivModelWhiteCube.PixivTags()
result.parseTags(response, tags)
else:
# from member id search by tags
print "Not supported yet"
else:
url = PixivHelper.generateSearchTagUrl(tags, i, title_caption,
wild_card, oldest_first,
start_date, end_date,
member_id,
self._config.r18mode)
PixivHelper.printAndLog('info', 'Looping... for ' + url)
response = self.open(url).read()
parse_search_page = BeautifulSoup(response)
if self._config.dumpTagSearchPage and self._config.enableDump:
dump_filename = PixivHelper.dumpHtml(url + ".html",
parse_search_page)
PixivHelper.printAndLog(
'info', "Dump tag search page to: " + dump_filename)
result = PixivModel.PixivTags()
if not member_id is None:
result.parseMemberTags(parse_search_page, member_id, tags)
else:
try:
result.parseTags(parse_search_page, tags)
except:
PixivHelper.dumpHtml(
"Dump for SearchTags " + tags + ".html",
search_page.get_data())
raise
parse_search_page.decompose()
del parse_search_page
return (result, response)
def getSearchTagPage(self,
tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1,
include_bookmark_data=False):
response = None
result = None
url = ''
if member_id is not None:
# from member id search by tags
(artist, response) = self.getMemberPage(member_id, current_page,
False, tags)
# convert to PixivTags
result = PixivModelWhiteCube.PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generateSearchTagUrl(tags, current_page,
title_caption, wild_card,
oldest_first, start_date,
end_date, member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for ' + url)
response = self.getPixivPage(url, returnParsed=False)
self.handleDebugTagSearchPage(response, url)
result = None
if member_id is not None:
result = PixivModel.PixivTags()
parse_search_page = BeautifulSoup(response)
result.parseMemberTags(parse_search_page, member_id, tags)
parse_search_page.decompose()
del parse_search_page
else:
try:
result = PixivModelWhiteCube.PixivTags()
result.parseTags(response, tags, current_page)
# parse additional information
if include_bookmark_data:
idx = 0
print("Retrieving bookmark information...", end=' ')
for image in result.itemList:
idx = idx + 1
print("\r", end=' ')
print(
"Retrieving bookmark information... [{0}] of [{1}]"
.format(idx, len(result.itemList)),
end=' ')
img_url = "https://www.pixiv.net/ajax/illust/{0}".format(
image.imageId)
response = self._get_from_cache(img_url)
if response is None:
try:
response = self.open_with_retry(
img_url).read()
except urllib2.HTTPError as ex:
if ex.code == 404:
response = ex.read()
self._put_to_cache(img_url, response)
image_info_js = json.loads(response)
image.bookmarkCount = int(
image_info_js["body"]["bookmarkCount"])
image.imageResponse = int(
image_info_js["body"]["responseCount"])
print("")
except BaseException:
PixivHelper.dumpHtml(
"Dump for SearchTags " + tags + ".html", response)
raise
return (result, response)
def getImagePage(self,
image_id,
parent=None,
from_bookmark=False,
bookmark_count=-1,
image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: %s", image_id)
if self._isWhitecube:
pass
## url = "https://www.pixiv.net/rpc/whitecube/index.php?mode=work_details_modal_whitecube&id={0}&tt={1}".format(image_id, self._whitecubeToken)
## response = self.open(url).read()
## self.handleDebugMediumPage(response, image_id)
## # PixivHelper.GetLogger().debug(response)
##
## image = PixivModelWhiteCube.PixivImage(image_id,
## response,
## parent,
## from_bookmark,
## bookmark_count,
## image_response_count,
## dateFormat=self._config.dateFormat)
## # overwrite artist info
## if from_bookmark:
## self.getMemberInfoWhitecube(image.originalArtist.artistId, image.originalArtist)
## else:
## self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
else:
url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(
image_id)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("globalInitData") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivModelWhiteCube.PixivImage(
image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(
image_id)
meta_response = self.open(ugoira_meta_url).read()
image.ParseUgoira(meta_response)
## PixivHelper.GetLogger().debug("animation.js")
## PixivHelper.GetLogger().debug(image.ugoira_data)
if parent is None:
if from_bookmark:
self.getMemberInfoWhitecube(
image.originalArtist.artistId,
image.originalArtist)
else:
self.getMemberInfoWhitecube(
image.artist.artistId, image.artist)
else:
parsed = BeautifulSoup(response)
image = PixivModel.PixivImage(
image_id,
parsed,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat)
if image.imageMode == "ugoira_view" or image.imageMode == "bigNew":
image.ParseImages(parsed)
parsed.decompose()
except:
PixivHelper.GetLogger().error("Respose data: \r\n %s",
response)
raise
return (image, response)
def getSearchTagPage(self,
tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1):
response = None
result = None
url = ''
if member_id is not None:
## if member_id is None:
## # from search page:
## # https://www.pixiv.net/rpc/whitecube/index.php?order=date&adult_mode=include&q=vocaloid&p=0&type=&mode=whitecube_search&s_mode=s_tag&scd=&size=&ratio=&like=&tools=&tt=4e2cdee233f1156231ee99da1e51a83c
## url = "https://www.pixiv.net/rpc/whitecube/index.php?q={0}".format(tags)
## url = url + "&adult_mode={0}".format("include")
## url = url + "&mode={0}".format("whitecube_search")
##
## # date ordering
## order = "date_d"
## if oldest_first:
## order = "date"
## url = url + "&order={0}".format(order)
##
## # search mode
## s_mode = "s_tag_full"
## if wild_card:
## s_mode = "s_tag"
## elif title_caption:
## s_mode = "s_tc"
## url = url + "&s_mode={0}".format(s_mode)
##
## # start/end date
## if start_date is not None:
## url = url + "&scd={0}".format(start_date)
## if end_date is not None:
## url = url + "&ecd={0}".format(end_date)
##
## url = url + "&p={0}".format(i)
## url = url + "&start_page={0}".format(start_page)
## url = url + "&tt={0}".format(self._whitecubeToken)
##
## PixivHelper.print_and_log('info', 'Looping for {0} ...'.format(url))
## response = self.open(url).read()
## self.handleDebugTagSearchPage(response, url)
##
## PixivHelper.GetLogger().debug(response)
## result = PixivModelWhiteCube.PixivTags()
## result.parseTags(response, tags)
## else:
# from member id search by tags
(artist, response) = self.getMemberPage(member_id, current_page,
False, tags)
# convert to PixivTags
result = PixivModelWhiteCube.PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generateSearchTagUrl(tags, current_page,
title_caption, wild_card,
oldest_first, start_date,
end_date, member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for ' + url)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugTagSearchPage(response, url)
parse_search_page = BeautifulSoup(response)
result = PixivModel.PixivTags()
if member_id is not None:
result.parseMemberTags(parse_search_page, member_id, tags)
else:
try:
result.parseTags(parse_search_page, tags)
except BaseException:
PixivHelper.dumpHtml(
"Dump for SearchTags " + tags + ".html", response)
raise
parse_search_page.decompose()
del parse_search_page
return (result, response)
from pathlib import Path
from flask import Flask, request, send_from_directory
from flask_restful import Api, Resource
import PixivConfig
import PixivModel as PM
PixivConfig.cd_script_dir()
pcfg = PixivConfig.PixivConfig('config.json')
app = Flask('PixivWebAPI')
api = Api(app)
db = PM.PixivDB(pcfg.database_uri)
class GetWorksCaption(Resource):
def get(self, works_id: int):
session = db.sessionmaker()
r = session.query(PM.WorksCaption).filter(
PM.WorksCaption.works_id == works_id).one_or_none()
if r:
return r.caption_text
api.add_resource(GetWorksCaption, '/api/getCaption/<int:works_id>')
@app.route('/image/<path:path>')
def send_stored_img(path):
return send_from_directory('./Test/stable/storage/works', path)