def testFanboxPost_url_embed(self):
reader = open('./test/fanbox_urlembed.json', 'r', encoding="utf-8")
p = reader.read()
reader.close()
js = json.loads(p)
dummy_artist = FanboxArtist(1, "kurikara", "kurikara")
# https://www.fanbox.cc/@kurikara/posts/4071336 => https://api.fanbox.cc/post.info?postId=4071336
post = FanboxPost(4071336, dummy_artist, js["body"], None)
self.assertIsNotNone(post)
# post-201946 article
self.assertEqual(post.imageId, 4071336)
self.assertTrue(len(post.imageTitle) > 0)
self.assertEqual(
post.coverImageUrl,
"https://pixiv.pximg.net/fanbox/public/images/post/4071336/cover/17bKPzVXhXtTz8dCeDMDg8qR.jpeg"
)
self.assertEqual(post.type, "article")
self.assertEqual(len(post.images), 1)
self.assertEqual(len(post.descriptionUrlList), 2)
# template = open('./template.html', 'r', encoding="utf-8").read()
# post.WriteHtml(template, False, "./4071336.html")
self.assertEqual(len(post.body_text), 3598)
def fanboxGetPost(self, post_id, artist=None):
self.fanbox_is_logged_in()
# https://fanbox.pixiv.net/api/post.info?postId=279561
# https://www.pixiv.net/fanbox/creator/104409/post/279561
p_url = f"https://api.fanbox.cc/post.info?postId={post_id}"
# referer doesn't seeem to be essential
p_referer = f"https://www.fanbox.cc/@{artist.creatorId if artist else ''}/posts/{post_id}"
PixivHelper.get_logger().debug('Getting post detail from %s', p_url)
p_req = mechanize.Request(p_url)
p_req.add_header('Accept', 'application/json, text/plain, */*')
p_req.add_header('Referer', p_referer)
p_req.add_header('Origin', 'https://www.fanbox.cc')
p_req.add_header('User-Agent', self._config.useragent)
p_res = self.open_with_retry(p_req)
p_response = p_res.read()
PixivHelper.get_logger().debug(p_response.decode('utf8'))
p_res.close()
js = demjson.decode(p_response)
if artist:
return js
else:
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
artist = FanboxArtist(js["body"]["user"]["userId"], js["body"]["creatorId"], js["body"]["user"]["name"])
self.fanboxUpdateArtistToken(artist)
post = FanboxPost(post_id, artist, js["body"], _tzInfo)
return post
def testFanboxNewApi2_Files(self):
p = open('./test/Fanbox_post_with_files.json', 'r',
encoding="utf-8").read()
js = demjson.decode(p)
result = FanboxPost(685832, None, js["body"])
self.assertIsNotNone(result)
self.assertEqual(result.imageId, 685832)
self.assertEqual(len(result.images), 1)
self.assertEqual(len(result.embeddedFiles), 2)
self.assertIsNotNone(result.coverImageUrl)
self.assertFalse(result.coverImageUrl in result.images)
def testFanboxNewApi2_MultiImages(self):
reader = open('./test/Fanbox_post_with_multi_images.json', 'r', encoding="utf-8")
p = reader.read()
reader.close()
js = demjson.decode(p)
result = FanboxPost(855025, None, js["body"])
self.assertIsNotNone(result)
self.assertEqual(result.imageId, 855025)
self.assertEqual(len(result.images), 2)
self.assertEqual(len(result.embeddedFiles), 3)
self.assertIsNotNone(result.coverImageUrl)
self.assertFalse(result.coverImageUrl in result.images)
def test_links_in_p_tags(self):
with open('./test/test_for_links_in_p_tags.json',
'r',
encoding="utf-8") as reader:
p = reader.read()
js = json.loads(p)
result = FanboxPost(6544246, None, js["body"])
self.assertIsNotNone(result)
test_string1 = "<a href='https://www.pixiv.net/fanbox/creator/6544246/post/407551'>{0}</a>".format(
u"Bleach: S\u014dsuke\u0027s Revenge Ch.2 "[0:29])
self.assertTrue(test_string1 in result.body_text)
temp_string = "H x H: The Plan to Wipe Out the Strongests "
test_string2 = "{0}<a href='{1}'>{2}</a>{3}<a href='{4}'>{5}</a>{6}".format(
temp_string[:5],
"https://www.pixiv.net/fanbox/creator/6544246/post/407881",
temp_string[5:15], temp_string[15:20], "#modified_for_test",
temp_string[20:30], temp_string[30:])
self.assertTrue(test_string2 in result.body_text)
def process_fanbox_post(caller, config, post: PixivModelFanbox.FanboxPost,
artist):
# caller function/method
# TODO: ideally to be removed or passed as argument
db = caller.__dbManager__
br = PixivBrowserFactory.getBrowser()
db.insertPost(artist.artistId, post.imageId, post.imageTitle,
post.feeRequired, post.worksDate, post.type)
post_files = []
flag_processed = False
if config.checkDBProcessHistory:
result = db.selectPostByPostId(post.imageId)
if result:
updated_date = result[5]
if updated_date is not None and post.updatedDateDatetime <= datetime_z.parse_datetime(
updated_date):
flag_processed = True
try:
if not post.is_restricted and not flag_processed:
br.fanboxUpdatePost(post)
if ((not post.is_restricted) or config.downloadCoverWhenRestricted
) and (not flag_processed) and config.downloadCover:
# cover image
if post.coverImageUrl:
# fake the image_url for filename compatibility, add post id and pagenum
fake_image_url = post.coverImageUrl.replace(
"{0}/cover/".format(post.imageId),
"{0}_".format(post.imageId))
filename = PixivHelper.make_filename(
config.filenameFormatFanboxCover,
post,
artistInfo=artist,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl=fake_image_url,
bookmark=None,
searchTags='',
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
filename = PixivHelper.sanitize_filename(
filename, config.rootDirectory)
post.linkToFile[post.coverImageUrl] = filename
print("Downloading cover from {0}".format(post.coverImageUrl))
print("Saved to {0}".format(filename))
referer = "https://www.pixiv.net/fanbox/creator/{0}/post/{1}".format(
artist.artistId, post.imageId)
# don't pass the post id and page number to skip db check
(result, filename) = PixivDownloadHandler.download_image(
caller,
post.coverImageUrl,
filename,
referer,
config.overwrite,
config.retry,
config.backupOldFile,
image=post,
download_from=PixivConstant.DOWNLOAD_FANBOX)
post_files.append((post.imageId, -1, filename))
PixivHelper.get_logger().debug("Download %s result: %s",
filename, result)
else:
PixivHelper.print_and_log(
"info",
"No Cover Image for post: {0}.".format(post.imageId))
if post.is_restricted:
PixivHelper.print_and_log(
"info", "Skipping post: {0} due to restricted post.".format(
post.imageId))
return
if flag_processed:
PixivHelper.print_and_log(
"info",
"Skipping post: {0} because it was downloaded before.".format(
post.imageId))
return
if post.images is None or len(post.images) == 0:
PixivHelper.print_and_log(
"info",
"No Image available in post: {0}.".format(post.imageId))
else:
current_page = 0
print("Image Count = {0}".format(len(post.images)))
for image_url in post.images:
# fake the image_url for filename compatibility, add post id and pagenum
fake_image_url = image_url.replace(
"{0}/".format(post.imageId),
"{0}_p{1}_".format(post.imageId, current_page))
filename = PixivHelper.make_filename(
config.filenameFormatFanboxContent,
post,
artistInfo=artist,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl=fake_image_url,
bookmark=None,
searchTags='',
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
filename = PixivHelper.sanitize_filename(
filename, config.rootDirectory)
post.linkToFile[image_url] = filename
referer = "https://www.pixiv.net/fanbox/creator/{0}/post/{1}".format(
artist.artistId, post.imageId)
print("Downloading image {0} from {1}".format(
current_page, image_url))
print("Saved to {0}".format(filename))
# filesize detection and overwrite issue
_oldvalue = config.alwaysCheckFileSize
config.alwaysCheckFileSize = False
# don't pass the post id and page number to skip db check
(result, filename) = PixivDownloadHandler.download_image(
caller,
image_url,
filename,
referer,
False, # config.overwrite somehow unable to get remote filesize
config.retry,
config.backupOldFile,
image=post,
download_from=PixivConstant.DOWNLOAD_FANBOX)
if result == PixivConstant.PIXIVUTIL_KEYBOARD_INTERRUPT:
raise KeyboardInterrupt()
post_files.append((post.imageId, current_page, filename))
PixivHelper.get_logger().debug("Download %s result: %s",
filename, result)
config.alwaysCheckFileSize = _oldvalue
current_page = current_page + 1
# Implement #447
filename = PixivHelper.make_filename(
config.filenameFormatFanboxInfo,
post,
artistInfo=artist,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl="{0}".format(post.imageId),
bookmark=None,
searchTags='',
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
filename = PixivHelper.sanitize_filename(filename,
config.rootDirectory)
if config.writeImageInfo:
post.WriteInfo(filename + ".txt")
if config.writeHtml:
if post.type == "article" or (
len(post.images) >= config.minImageCountForNonArticle and
len(post.body_text) > config.minTextLengthForNonArticle):
html_template = PixivConstant.HTML_TEMPLATE
if os.path.isfile("template.html"):
reader = PixivHelper.open_text_file("template.html")
html_template = reader.read()
reader.close()
post.WriteHtml(html_template, config.useAbsolutePathsInHtml,
filename + ".html")
if config.writeUrlInDescription:
PixivHelper.write_url_in_description(post,
config.urlBlacklistRegex,
config.urlDumpFilename)
finally:
if len(post_files) > 0:
db.insertPostImages(post_files)
db.updatePostUpdateDate(post.imageId, post.updatedDate)