def testTagsSkipShowcase(self):
path = './test/test-tags-search-skip-showcase.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
def testTagsSkipShowcase(self):
path = './test/test-tags-search-skip-showcase.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
def testTagsSearchPartialLast(self):
path = './test/test-tags-search-partial-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(image.itemList[-1].imageId, 15060554)
self.assertEqual(image.isLastPage, True)
def testTagsSearchPartial(self):
path = './test/test-tags-search-partial.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
def testTagsSearchPartial(self):
path = './test/test-tags-search-partial.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
def testTagsSkipShowcase(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-skip-showcase.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 20)
def testTagsSearchPartialLast(self):
path = './test/test-tags-search-partial-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(image.itemList[-1].imageId, 15060554)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExact(self):
path = './test/test-tags-search-exact.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
## for img in image.itemList:
## print(img.imageId)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
def testTagsSearchExact1(self):
path = './test/test-tags-search-exact2.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 2283)
def testTagsMemberSearchLast(self):
path = './test/test-tags-member-search-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
# self.assertEqual(len(image.itemList), 10)
self.assertEqual(image.itemList[-1].imageId, 1804545)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExact(self):
br = Browser()
path = "file:///" + os.path.abspath("./test/test-tags-search-exact.htm").replace(os.sep, "/")
p = br.open(path, "r")
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.isLastPage, False)
def testTagsSearchPartialLast(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-partial-last.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(image.itemList[-1].imageId, 15060554)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExact1(self):
path = './test/test-tags-search-exact2.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 2295)
def testTagsMemberSearchLast(self):
path = './test/test-tags-member-search-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
# self.assertEqual(len(image.itemList), 10)
self.assertEqual(image.itemList[-1].imageId, 1804545)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExactLast(self):
path = './test/test-tags-search-exact-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
print(image.itemList[-1])
self.assertEqual(image.itemList[-1].imageId, 544700)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExact(self):
path = './test/test-tags-search-exact.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
## for img in image.itemList:
## print(img.imageId)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
def testTagsSearchPartial(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-partial.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.isLastPage, False)
def testTagsSearchExactLast(self):
path = './test/test-tags-search-exact-last.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
# self.assertEqual(len(image.itemList), 3)
self.assertEqual(image.itemList[-1].imageId, 544700)
self.assertEqual(image.isLastPage, True)
def testTagsSearchParseDetails(self):
path = './test/test-tags-search-exact-parse_details.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
# self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[-1].imageId, 33815932)
self.assertGreater(image.itemList[-1].bookmarkCount, 4)
self.assertEqual(image.itemList[-1].imageResponse, 0)
def testTagsSearchParseDetails(self):
path = './test/test-tags-search-exact-parse_details.htm'
p = open(path, 'r')
response = p.read()
tags = ''
current_page = 1
image = PixivTags()
image.parseTags(response, tags, current_page)
self.assertEqual(image.itemList[-1].imageId, 33815932)
def testTagsSkipShowcase(self):
path = './test/test-tags-search-skip-showcase.htm'
p = open(path, 'r')
response = p.read()
tags = 'K-On!'
current_page = 1
image = PixivTags()
image.parseTags(response, tags, current_page)
self.assertEqual(len(image.itemList), 60)
def testTagsSearchParseDetails(self):
path = './test/test-tags-search-exact-parse_details.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
# self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[-1].imageId, 33815932)
self.assertGreater(image.itemList[-1].bookmarkCount, 4)
self.assertEqual(image.itemList[-1].imageResponse, 0)
def testTagsMemberSearchLast(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-member-search-last.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page)
##self.assertEqual(len(image.itemList), 10)
self.assertEqual(image.itemList[-1].imageId, 1804545)
self.assertEqual(image.isLastPage, True)
def testTagsSearchExact(self):
path = './test/test-tags-search-exact.htm'
p = open(path, 'r')
response = p.read()
tags = ''
current_page = 1
image = PixivTags()
image.parseTags(response, tags, current_page)
self.assertEqual(len(image.itemList), 60)
self.assertEqual(image.isLastPage, False)
def testTagsSearchExact(self):
br = Browser()
path = 'file:///' + os.path.abspath(
'./test/test-tags-search-exact.htm').replace(os.sep, '/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
for img in image.itemList:
print img.imageId
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
def testTagsSearchExact1(self):
br = Browser()
path = 'file:///' + os.path.abspath(
u'./test/test-tags-search-exact2.htm').replace(os.sep, '/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 40)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 2285)
def testTagsSearchPartial(self):
path = './test/test-tags-search-partial.htm'
p = open(path, 'r')
response = p.read()
tags = '%E3%81%93%E3%81%AE%E4%B8%AD%E3%81%AB1%E4%BA%BA%E3%80%81%E5%A6%B9%E3%81%8C%E3%81%84%E3%82%8B!'
current_page = 1
image = PixivTags()
image.parseTags(response, tags, current_page)
self.assertEqual(len(image.itemList), 60)
self.assertEqual(image.isLastPage, False)
def testTagsSearchParseDetails(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-exact-parse_details.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
##self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[-1].imageId, 33815932)
self.assertEqual(image.itemList[-1].bookmarkCount, 4)
self.assertEqual(image.itemList[-1].imageResponse, 0)
def testTagsSearchPartialLast(self):
path = './test/test-tags-search-partial-last.htm'
p = open(path, 'r')
response = p.read()
tags = '%E3%81%93%E3%81%AE%E4%B8%AD%E3%81%AB1%E4%BA%BA%E3%80%81%E5%A6%B9%E3%81%8C%E3%81%84%E3%82%8B!'
current_page = 4
image = PixivTags()
image.parseTags(response, tags, current_page)
self.assertEqual(image.itemList[-1].imageId, 15060554)
self.assertEqual(image.isLastPage, True)
def testTagsMemberSearch(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-member-search.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page)
self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[0].imageId, 25757869)
self.assertEqual(image.itemList[19].imageId, 14818847)
self.assertEqual(image.isLastPage, False)
def testTagsMemberSearch(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-member-search.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page)
self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[0].imageId, 25757869)
self.assertEqual(image.itemList[19].imageId, 14818847)
self.assertEqual(image.isLastPage, False)
def testTagsMemberSearch(self):
path = './test/test-tags-member-search.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
self.assertEqual(len(image.itemList), 40)
# self.assertEqual(image.itemList[0].imageId, 53977340)
# self.assertEqual(image.itemList[19].imageId, 45511597)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 73)
def testTagsMemberSearch(self):
path = './test/test-tags-member-search.htm'
p = open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
self.assertEqual(len(image.itemList), 40)
# self.assertEqual(image.itemList[0].imageId, 53977340)
# self.assertEqual(image.itemList[19].imageId, 45511597)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 73)
def testTagsSearchExactLast(self):
path = './test/test-tags-search-exact-last.htm'
p = open(path, 'r')
response = p.read()
tags = ''
current_page = 47
image = PixivTags()
image.parseTags(response, tags, current_page)
print(image.itemList[-1])
self.assertEqual(image.itemList[-1].imageId, 544700)
self.assertEqual(image.isLastPage, True)
def testTagsMemberSearch(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-member-search.htm').replace(os.sep, '/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
self.assertEqual(len(image.itemList), 40)
# self.assertEqual(image.itemList[0].imageId, 53977340)
# self.assertEqual(image.itemList[19].imageId, 45511597)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 66)
def testTagsMemberSearch(self):
br = Browser()
path = 'file:///' + os.path.abspath(
'./test/test-tags-member-search.htm').replace(os.sep, '/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseMemberTags(page, 313631)
self.assertEqual(len(image.itemList), 40)
# self.assertEqual(image.itemList[0].imageId, 53977340)
# self.assertEqual(image.itemList[19].imageId, 45511597)
self.assertEqual(image.isLastPage, False)
self.assertEqual(image.availableImages, 70)
def testTagsSearchExactLast(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-exact-last.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 4)
self.assertEqual(image.itemList[0].imageId, 21618970)
self.assertEqual(image.itemList[0].bookmarkCount, -1)
self.assertEqual(image.itemList[0].imageResponse, -1)
self.assertEqual(image.itemList[3].imageId, 15060554)
self.assertEqual(image.itemList[3].bookmarkCount, 1)
self.assertEqual(image.itemList[3].imageResponse, -1)
self.assertEqual(image.isLastPage, True)
def testTagsSearchPartial(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-partial.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 20)
self.assertEqual(image.itemList[0].imageId, 27792358)
self.assertEqual(image.itemList[0].bookmarkCount, 2)
self.assertEqual(image.itemList[0].imageResponse, -1)
self.assertEqual(image.itemList[19].imageId, 27110688)
self.assertEqual(image.itemList[19].bookmarkCount, -1)
self.assertEqual(image.itemList[19].imageResponse, -1)
self.assertEqual(image.isLastPage, False)
def testTagsSearchParseDetails(self):
br = Browser()
path = 'file:///' + os.path.abspath('./test/test-tags-search-exact-parse_details.htm').replace(os.sep,'/')
p = br.open(path, 'r')
page = BeautifulSoup(p.read())
image = PixivTags()
image.parseTags(page)
self.assertEqual(len(image.itemList), 20)
## http://www.pixiv.net/member_illust.php?mode=medium&illust_id=26563564
self.assertEqual(image.itemList[0].imageId, 26563564)
self.assertEqual(image.itemList[0].bookmarkCount, 1)
## http://www.pixiv.net/member_illust.php?mode=medium&illust_id=26557089
self.assertEqual(image.itemList[1].imageId, 26557089)
self.assertEqual(image.itemList[1].bookmarkCount, 3)
self.assertEqual(image.itemList[1].imageResponse, 14)
## http://www.pixiv.net/member_illust.php?mode=medium&illust_id=26538917
self.assertEqual(image.itemList[5].imageId, 26538917)
self.assertEqual(image.itemList[5].bookmarkCount, -1)
self.assertEqual(image.itemList[5].imageResponse, -1)
def getSearchTagPage(self, tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1,
include_bookmark_data=False):
response = None
result = None
url = ''
if member_id is not None:
# from member id search by tags
(artist, response) = self.getMemberPage(member_id, current_page, False, tags)
# convert to PixivTags
result = PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generateSearchTagUrl(tags, current_page,
title_caption,
wild_card,
oldest_first,
start_date,
end_date,
member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for ' + url)
response = self.getPixivPage(url, returnParsed=False)
self.handleDebugTagSearchPage(response, url)
result = None
if member_id is not None:
result = PixivTags()
parse_search_page = BeautifulSoup(response)
result.parseMemberTags(parse_search_page, member_id, tags)
parse_search_page.decompose()
del parse_search_page
else:
try:
result = PixivTags()
result.parseTags(response, tags, current_page)
# parse additional information
if include_bookmark_data:
idx = 0
print("Retrieving bookmark information...", end=' ')
for image in result.itemList:
idx = idx + 1
print("\r", end=' ')
print("Retrieving bookmark information... [{0}] of [{1}]".format(idx, len(result.itemList)), end=' ')
img_url = "https://www.pixiv.net/ajax/illust/{0}".format(image.imageId)
response = self._get_from_cache(img_url)
if response is None:
try:
response = self.open_with_retry(img_url).read()
except urllib2.HTTPError as ex:
if ex.code == 404:
response = ex.read()
self._put_to_cache(img_url, response)
image_info_js = json.loads(response)
image.bookmarkCount = int(image_info_js["body"]["bookmarkCount"])
image.imageResponse = int(image_info_js["body"]["responseCount"])
print("")
except BaseException:
PixivHelper.dumpHtml("Dump for SearchTags " + tags + ".html", response)
raise
return (result, response)
