def login(self, username, password):
try:
PixivHelper.printAndLog("info", "Logging in...")
url = "https://accounts.pixiv.net/login"
page = self.open(url)
# get the post key
parsed = BeautifulSoup(page)
init_config = parsed.find("input", attrs={"id": "init-config"})
js_init_config = json.loads(init_config["value"])
data = {}
data["pixiv_id"] = username
data["password"] = password
# data['captcha'] = ''
# data['g_recaptcha_response'] = ''
data["return_to"] = "http://www.pixiv.net"
data["lang"] = "en"
data["post_key"] = js_init_config["pixivAccount.postKey"]
data["source"] = "pc"
request = urllib2.Request("https://accounts.pixiv.net/api/login?lang=en", urllib.urlencode(data))
response = self.open(request)
return self.processLoginResult(response)
except:
PixivHelper.printAndLog("error", "Error at login(): " + str(sys.exc_info()))
raise
def cleanUp(self):
anim_ext = ['.zip', '.gif', '.apng', '.ugoira', '.webm']
try:
print("Start clean-up operation.")
print("Selecting all images, this may take some times.")
c = self.conn.cursor()
c.execute('''SELECT image_id, save_name from pixiv_master_image''')
print("Checking images.")
for row in c:
# Issue 340
filename = row[1]
fileExists = False
if filename is not None and len(filename) > 0:
if os.path.exists(filename):
continue
for ext in anim_ext:
# check filename in db against all combination possible filename in disk
if filename.endswith(ext):
base_filename = filename.rsplit(ext, 1)[0]
if self.checkFilenames(base_filename, anim_ext):
fileExists = True
break
if not fileExists:
PixivHelper.safePrint("Missing: {0} at {1}".format(row[0], row[1]))
self.deleteImage(row[0])
self.conn.commit()
except BaseException:
print('Error at cleanUp():', str(sys.exc_info()))
print('failed')
raise
finally:
c.close()
def printMemberList(self, isDeleted=False):
print 'Printing member list:'
try:
c = self.conn.cursor()
c.execute('''SELECT * FROM pixiv_master_member
WHERE is_deleted = ?
ORDER BY member_id''', (int(isDeleted), ))
print '%10s %25s %25s %20s %20s %10s %s' % ('member_id','name','save_folder','created_date','last_update_date','last_image','is_deleted')
i = 0
for row in c:
#for string in row:
# print '\t',
# PixivHelper.safePrint(unicode(string), False)
PixivHelper.safePrint('%10d %#25s %#25s %20s %20s %10d %5s' % (row[0], unicode(row[1]).strip(), row[2], row[3], row[4], row[5], row[6]))
#print ''
i = i + 1
if i == 79:
select = raw_input('Continue [y/n]? ')
if select == 'n':
break
else :
print 'member_id\tname\tsave_folder\tcreated_date\tlast_update_date\tlast_image'
i = 0
except:
print 'Error at printList():',str(sys.exc_info())
print 'failed'
raise
finally:
c.close()
print 'done.'
def login(self, username, password):
try:
PixivHelper.print_and_log('info', 'Logging in...')
url = "https://accounts.pixiv.net/login"
page = self.open_with_retry(url)
# get the post key
parsed = BeautifulSoup(page)
js_init_config = self._getInitConfig(parsed)
data = {}
data['pixiv_id'] = username
data['password'] = password
data['captcha'] = ''
data['g_recaptcha_response'] = ''
data['return_to'] = 'https://www.pixiv.net'
data['lang'] = 'en'
data['post_key'] = js_init_config["pixivAccount.postKey"]
data['source'] = "accounts"
data['ref'] = ''
request = urllib2.Request("https://accounts.pixiv.net/api/login?lang=en", urllib.urlencode(data))
response = self.open_with_retry(request)
return self.processLoginResult(response, username, password)
except BaseException:
PixivHelper.print_and_log('error', 'Error at login(): {0}'.format(sys.exc_info()))
raise
def WriteInfo(self, filename):
info = None
try:
# Issue #421 ensure subdir exists.
PixivHelper.makeSubdirs(filename)
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".txt", 'wb', encoding='utf-8')
PixivHelper.GetLogger().exception("Error when saving image info: {0}, file is saved to: {1}.txt".format(filename, self.imageId))
info.write(u"ArtistID = {0}\r\n".format(self.parent.artistId))
info.write(u"ArtistName = {0}\r\n".format(self.parent.artistName))
info.write(u"ImageID = {0}\r\n".format(self.imageId))
info.write(u"Title = {0}\r\n".format(self.imageTitle))
info.write(u"Caption = {0}\r\n".format(self.body_text))
# info.write(u"Tags = " + ", ".join(self.imageTags) + "\r\n")
if self.is_restricted:
info.write(u"Image Mode = {0}, Restricted\r\n".format(self.type))
else:
info.write(u"Image Mode = {0}\r\n".format(self.type))
info.write(u"Pages = {0}\r\n".format(self.imageCount))
info.write(u"Date = {0}\r\n".format(self.worksDate))
# info.write(u"Resolution = " + self.worksResolution + "\r\n")
# info.write(u"Tools = " + self.worksTools + "\r\n")
info.write(u"Like Count = {0}\r\n".format(self.likeCount))
info.write(u"Link = https://www.pixiv.net/fanbox/creator/{0}/post/{1}\r\n".format(self.parent.artistId, self.imageId))
# info.write("Ugoira Data = " + str(self.ugoira_data) + "\r\n")
if len(self.embeddedFiles) > 0:
info.write("Urls =\r\n")
for link in self.embeddedFiles:
info.write(" - {0}\r\n".format(link))
info.close()
def login(self):
oauth_response = None
need_relogin = True
if self._refresh_token is not None:
PixivHelper.GetLogger().info("Login to OAuth using refresh token.")
oauth_response = requests.post(self._url,
self._get_values_for_refresh(),
headers=self._get_default_headers(),
proxies=self._proxies,
verify=self._validate_ssl)
if oauth_response.status_code == 200:
need_relogin = False
else:
PixivHelper.GetLogger().info("OAuth Refresh Token invalid, Relogin needed.")
if need_relogin:
oauth_response = self.login_with_username_and_password()
PixivHelper.GetLogger().debug("{0}: {1}".format(oauth_response.status_code, oauth_response.text))
if oauth_response.status_code == 200:
info = json.loads(oauth_response.text)
self._refresh_token = info["response"]["refresh_token"]
self._access_token = info["response"]["access_token"]
elif oauth_response.status_code == 400:
PixivHelper.print_and_log('error', oauth_response.text)
raise PixivException("Failed to login using OAuth", PixivException.OAUTH_LOGIN_ISSUE, oauth_response.text)
return oauth_response
def replaceRootPath(self):
oldPath = raw_input("Old Path to Replace = ")
PixivHelper.safePrint("Replacing " + oldPath + " to " + self.__config__.rootDirectory)
cont = raw_input("continue[y/n]?") or 'n'
if cont != "y":
print "Aborted"
return
try:
print "Start replace Root Path operation."
print "Updating images, this may take some times."
c = self.conn.cursor()
c.execute('''UPDATE pixiv_master_image
SET save_name = replace(save_name, ?, ?)
WHERE save_name like ?''', (oldPath, self.__config__.rootDirectory, oldPath + "%", ))
print "Updated image:", c.rowcount
print "Done"
except:
print 'Error at replaceRootPath():',str(sys.exc_info())
print 'failed'
raise
finally:
c.close()
def testCreateMangaFilename(self):
p = open('./test/test-image-manga.htm', 'r')
page = BeautifulSoup(p.read())
imageInfo = PixivImage(28820443, page)
imageInfo.imageCount = 100
page.decompose()
del page
# cross check with json value for artist info
js_file = open('./test/detail-554800.json', 'r')
js = json.load(js_file)
self.assertEqual(imageInfo.artist.artistId, str(js["user"]["id"]))
self.assertEqual(imageInfo.artist.artistToken, js["user"]["account"])
self.assertEqual(imageInfo.artist.artistAvatar, js["user"]["profile_image_urls"]["medium"].replace("_170", ""))
nameFormat = '%member_token% (%member_id%)\\%urlFilename% %page_number% %works_date_only% %works_res% %works_tools% %title%'
expected = unicode(u'maidoll (554800)\\28865189_p0 001 07/22/12 Multiple images: 2P C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p0.jpg')
# print(result)
self.assertEqual(result, expected)
expected = unicode(u'maidoll (554800)\\28865189_p14 015 07/22/12 Multiple images: 2P C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p14.jpg')
# print(result)
self.assertEqual(result, expected)
expected = unicode(u'maidoll (554800)\\28865189_p921 922 07/22/12 Multiple images: 2P C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p921.jpg')
# print(result)
self.assertEqual(result, expected)
def printMembersByLastDownloadDate(self, difference):
rows = self.selectMembersByLastDownloadDate(difference)
for row in rows:
for string in row:
print ' ',
PixivHelper.safePrint(unicode(string), False)
print '\n'
def fanboxGetSupportedUsers(self):
''' get all supported users from the list from https://www.pixiv.net/ajax/fanbox/support'''
url = 'https://www.pixiv.net/ajax/fanbox/support'
PixivHelper.print_and_log('info', 'Getting supported artists from ' + url)
# read the json response
response = self.open_with_retry(url).read()
result = Fanbox(response)
return result
def getMyId(self, parsed):
''' Assume from main page '''
# pixiv.user.id = "189816";
temp = re.findall(r"pixiv.user.id = \"(\d+)\";", unicode(parsed))
if temp is not None:
self._myId = int(temp[0])
PixivHelper.print_and_log('info', 'My User Id: {0}.'.format(self._myId))
else:
PixivHelper.print_and_log('info', 'Unable to get User Id')
def get_user_info(self, userid):
url = 'https://app-api.pixiv.net/v1/user/detail?user_id={0}'.format(userid)
user_info = requests.get(url,
None,
headers=self._get_headers_with_bearer(),
proxies=self._proxies,
verify=self._validate_ssl)
if user_info.status_code == 404:
PixivHelper.print_and_log('error', user_info.text)
return user_info
def getSearchTagPage(self, tags,
current_page,
wild_card=True,
title_caption=False,
start_date=None,
end_date=None,
member_id=None,
oldest_first=False,
start_page=1):
response = None
result = None
url = ''
if member_id is not None:
# from member id search by tags
(artist, response) = self.getMemberPage(member_id, current_page, False, tags)
# convert to PixivTags
result = PixivModelWhiteCube.PixivTags()
result.parseMemberTags(artist, member_id, tags)
else:
# search by tags
url = PixivHelper.generateSearchTagUrl(tags, current_page,
title_caption,
wild_card,
oldest_first,
start_date,
end_date,
member_id,
self._config.r18mode)
PixivHelper.print_and_log('info', 'Looping... for ' + url)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugTagSearchPage(response, url)
parse_search_page = BeautifulSoup(response)
result = PixivModel.PixivTags()
if member_id is not None:
result.parseMemberTags(parse_search_page, member_id, tags)
else:
try:
result.parseTags(parse_search_page, tags)
except BaseException:
PixivHelper.dumpHtml("Dump for SearchTags " + tags + ".html", response)
raise
parse_search_page.decompose()
del parse_search_page
return (result, response)
def testcreateAvatarFilenameFormatWithSubfolderNoRootDir(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = ''
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%member_token% (%member_id%)' + os.sep + '%image_id% - %title% - %tags%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
PixivHelper.setConfig(_config)
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, self.currPath + os.sep + u'kirabara29 (1107124)' + os.sep + 'folder.jpg')
def __init__(self, target = '', config=None):
if target is None or len(target) == 0:
target = script_path + os.sep + "db.sqlite"
PixivHelper.printAndLog('info',"using default DB Path: " + target)
else:
PixivHelper.printAndLog('info',"using custom DB Path: " + target)
self.conn = sqlite3.connect(target)
if config is not None:
self.__config__ = config
else:
self.__config__ = PixivConfig.PixivConfig()
self.__config__.loadConfig()
def processLoginResult(self, response):
PixivHelper.GetLogger().info('Logging in, return url: ' + response.geturl())
## failed login will return to either of these page:
## http://www.pixiv.net/login.php
## https://www.secure.pixiv.net/login.php
if response.geturl().find('pixiv.net/login.php') == -1:
PixivHelper.printAndLog('info','Logged in')
## write back the new cookie value
for cookie in self._ua_handlers['_cookies'].cookiejar:
if cookie.name == 'PHPSESSID':
PixivHelper.printAndLog('info', 'new cookie value: ' + str(cookie.value))
self._config.cookie = cookie.value
self._config.writeConfig(path=self._config.configFileLocation)
break
return True
else:
errors = self.parseLoginError(response)
if len(errors) > 0:
for error in errors:
if error.string is not None:
PixivHelper.printAndLog('error', 'Server Reply: ' + error.string)
else:
PixivHelper.printAndLog('info', 'Wrong username or password.')
return False
def simple_from_images():
filename_format = '%title%_%urlFilename%'
filename = PixivHelper.makeFilename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitizeFilename(filename, root_dir)
self.assertEqual(filename, root_dir + os.sep + u"アスナさん０２_136761_p0_hcXl48iORoJykmrR3zPZEoUu.jpeg")
def getMemberPage(self, member_id, page=1, bookmark=False, tags=None):
artist = None
response = None
if tags is not None:
tags = PixivHelper.encode_tags(tags)
else:
tags = ''
limit = 48
offset = (page - 1) * limit
need_to_slice = False
if bookmark:
# https://www.pixiv.net/ajax/user/1039353/illusts/bookmarks?tag=&offset=0&limit=24&rest=show
url = 'https://www.pixiv.net/ajax/user/{0}/illusts/bookmarks?tag={1}&offset={2}&limit={3}&rest=show'.format(member_id, tags, offset, limit)
else:
# https://www.pixiv.net/ajax/user/1813972/illusts/tag?tag=Fate%2FGrandOrder?offset=0&limit=24
# https://www.pixiv.net/ajax/user/1813972/manga/tag?tag=%E3%83%A1%E3%82%A4%E3%82%AD%E3%83%B3%E3%82%B0?offset=0&limit=24
# https://www.pixiv.net/ajax/user/5238/illustmanga/tag?tag=R-18&offset=0&limit=48
# https://www.pixiv.net/ajax/user/1813972/profile/all
url = None
if len(tags) > 0:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format(member_id, tags, offset, limit)
elif self._config.r18mode:
url = 'https://www.pixiv.net/ajax/user/{0}/illustmanga/tag?tag={1}&offset={2}&limit={3}'.format(member_id, 'R-18', offset, limit)
else:
url = 'https://www.pixiv.net/ajax/user/{0}/profile/all'.format(member_id)
need_to_slice = True
PixivHelper.print_and_log('info', 'Member Url: ' + url)
if url is not None:
# cache the response
response = self._get_from_cache(url)
if response is None:
try:
response = self.open_with_retry(url).read()
except urllib2.HTTPError as ex:
if ex.code == 404:
response = ex.read()
self._put_to_cache(url, response)
PixivHelper.GetLogger().debug(response)
artist = PixivModelWhiteCube.PixivArtist(member_id, response, False, offset, limit)
artist.reference_image_id = artist.imageList[0] if len(artist.imageList) > 0 else 0
self.getMemberInfoWhitecube(member_id, artist, bookmark)
if artist.haveImages and need_to_slice:
artist.imageList = artist.imageList[offset:offset + limit]
return (artist, response)
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename) :
raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 0
for line in reader:
lineNo = lineNo + 1
originalLine = line
if line.startswith('#') or len(line) < 1:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
try:
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
except:
PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
def more_format():
# from images
filename_format = '%member_id%' + os.sep + '%image_id%_p%page_index%_%title%_%urlFilename%_%works_date%'
filename = PixivHelper.makeFilename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitizeFilename(filename, root_dir)
self.assertEqual(filename, root_dir + os.sep + u"15521131" + os.sep + u"136761_p0_アスナさん０２_136761_p0_hcXl48iORoJykmrR3zPZEoUu_2018-08-26 20_28_16.jpeg")
def ParseWorksData(self, page):
temp = page.find(attrs={'class':'meta'}).findAll('li')
#07/22/2011 03:09|512×600|RETAS STUDIO
#07/26/2011 00:30|Manga 39P|ComicStudio 鉛筆 つけペン
#1/05/2011 07:09|723×1023|Photoshop SAI [ R-18 ]
#2013年3月16日 06:44 | 800×1130 | Photoshop ComicStudio | R-18
#2013年12月14日 19:00 855×1133 PhotoshopSAI
self.worksDate = PixivHelper.toUnicode(temp[0].string, encoding=sys.stdin.encoding).replace(u'/', u'-')
if self.worksDate.find('-') > -1:
try:
self.worksDateDateTime = datetime.datetime.strptime(self.worksDate, u'%m-%d-%Y %H:%M')
except ValueError as ve:
PixivHelper.GetLogger().exception('Error when parsing datetime: {0} for imageId {1}'.format(self.worksDate, self.imageId))
self.worksDateDateTime = datetime.datetime.strptime(self.worksDate.split(" ")[0], u'%Y-%m-%d')
else:
tempDate = self.worksDate.replace(u'年', '-').replace(u'月','-').replace(u'日', '')
self.worksDateDateTime = datetime.datetime.strptime(tempDate, '%Y-%m-%d %H:%M')
self.worksResolution = unicode(temp[1].string).replace(u'×',u'x')
toolsTemp = page.find(attrs={'class':'meta'}).find(attrs={'class':'tools'})
if toolsTemp!= None and len(toolsTemp) > 0:
tools = toolsTemp.findAll('li')
for tool in tools:
self.worksTools = self.worksTools + ' ' + unicode(tool.string)
self.worksTools = self.worksTools.strip()
def ParseWorksData(self, page):
temp = page.find(attrs={"class": "meta"}).findAll("li")
# 07/22/2011 03:09|512×600|RETAS STUDIO
# 07/26/2011 00:30|Manga 39P|ComicStudio 鉛筆 つけペン
# 1/05/2011 07:09|723×1023|Photoshop SAI [ R-18 ]
# 2013年3月16日 06:44 | 800×1130 | Photoshop ComicStudio | R-18
# 2013年12月14日 19:00 855×1133 PhotoshopSAI
self.worksDate = PixivHelper.toUnicode(temp[0].string, encoding=sys.stdin.encoding).replace(u"/", u"-")
if self.worksDate.find("-") > -1:
try:
self.worksDateDateTime = datetime.datetime.strptime(self.worksDate, u"%m-%d-%Y %H:%M")
except ValueError as ve:
PixivHelper.GetLogger().exception(
"Error when parsing datetime: {0} for imageId {1}".format(self.worksDate, self.imageId)
)
self.worksDateDateTime = datetime.datetime.strptime(self.worksDate.split(" ")[0], u"%Y-%m-%d")
else:
tempDate = self.worksDate.replace(u"年", "-").replace(u"月", "-").replace(u"日", "")
self.worksDateDateTime = datetime.datetime.strptime(tempDate, "%Y-%m-%d %H:%M")
self.worksResolution = unicode(temp[1].string).replace(u"×", u"x")
toolsTemp = page.find(attrs={"class": "meta"}).find(attrs={"class": "tools"})
if toolsTemp != None and len(toolsTemp) > 0:
tools = toolsTemp.findAll("li")
for tool in tools:
self.worksTools = self.worksTools + " " + unicode(tool.string)
self.worksTools = self.worksTools.strip()
def loginHttps(self, username, password):
try:
PixivHelper.printAndLog('info', 'Log in using secure form.')
self.open(PixivConstant.PIXIV_URL_SSL)
self.select_form(predicate=lambda f: f.attrs.get('action', None) == '/login.php')
self['pixiv_id'] = username
self['pass'] = password
if self._config.keepSignedIn:
self.find_control('skip').items[0].selected = True
response = self.submit()
return self.processLoginResult(response, )
except:
PixivHelper.printAndLog('error', 'Error at loginHttps(): ' + str(sys.exc_info()))
raise
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename) :
raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION)
reader = PixivHelper.OpenTextFile(filename)
for line in reader:
if line.startswith('#') or len(line) < 1:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
try:
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
except:
PixivHelper.safePrint('Invalid line: '+line)
(exType, value, traceback) = sys.exc_info()
print 'Error at PixivListItem.parseList():', exType, value
reader.close()
return l
def getImagePage(self, image_id, parent=None, from_bookmark=False,
bookmark_count=-1, image_response_count=-1):
image = None
response = None
PixivHelper.GetLogger().debug("Getting image page: %s", image_id)
url = "https://www.pixiv.net/member_illust.php?mode=medium&illust_id={0}".format(image_id)
# response = self.open(url).read()
response = self.getPixivPage(url, returnParsed=False).read()
self.handleDebugMediumPage(response, image_id)
# Issue #355 new ui handler
image = None
try:
if response.find("globalInitData") > 0:
PixivHelper.print_and_log('debug', 'New UI Mode')
# Issue #420
_tzInfo = None
if self._config.useLocalTimezone:
_tzInfo = PixivHelper.LocalUTCOffsetTimezone()
image = PixivModelWhiteCube.PixivImage(image_id,
response,
parent,
from_bookmark,
bookmark_count,
image_response_count,
dateFormat=self._config.dateFormat,
tzInfo=_tzInfo)
if image.imageMode == "ugoira_view":
ugoira_meta_url = "https://www.pixiv.net/ajax/illust/{0}/ugoira_meta".format(image_id)
meta_response = self.open_with_retry(ugoira_meta_url).read()
image.ParseUgoira(meta_response)
if parent is None:
if from_bookmark:
image.originalArtist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.originalArtist.artistId, image.originalArtist)
else:
image.artist.reference_image_id = image_id
self.getMemberInfoWhitecube(image.artist.artistId, image.artist)
except:
PixivHelper.GetLogger().error("Respose data: \r\n %s", response)
raise
return (image, response)
def cover_more_format():
# https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/96862/cover/6SRpcQwIUuJdeZbhn5q85l9x.jpeg
fake_image_url = post.coverImageUrl.replace("{0}/cover/".format(post.imageId), "{0}_".format(post.imageId))
print(fake_image_url)
filename_format = '%member_id%' + os.sep + '%image_id%_%title%_%urlFilename%_%works_date%'
filename = PixivHelper.makeFilename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitizeFilename(filename, root_dir)
self.assertEqual(filename, root_dir + os.sep + u"15521131" + os.sep + u"136761_アスナさん０２_136761_OqhhcslOfbzZpHyTfJNtnIWm_2018-08-26 20_28_16.jpeg")
def loginHttps(self, username, password):
try:
PixivHelper.printAndLog('info', 'Log in using secure form.')
req = self._makeRequest(PixivConstant.PIXIV_URL_SSL)
self.open(req)
self.select_form(nr=PixivConstant.PIXIV_FORM_NUMBER_SSL)
self['pixiv_id'] = username
self['pass'] = password
if self._config.keepSignedIn:
self.find_control('skip').items[0].selected = True
response = self.submit()
return self.processLoginResult(response, )
except:
PixivHelper.printAndLog('error', 'Error at pixiv_login_ssl(): ' + str(sys.exc_info()))
raise
def fanboxGetPostsFromArtist(self, artist_id, next_url=""):
''' get all posts from the supported user from https://www.pixiv.net/ajax/fanbox/creator?userId=15521131 '''
if next_url is None or next_url == "":
url = "https://www.pixiv.net/ajax/fanbox/creator?userId={0}".format(artist_id)
else:
url = "https://www.pixiv.net" + next_url
PixivHelper.print_and_log('info', 'Getting posts from ' + url)
response = self.open(url).read()
result = FanboxArtist(artist_id, response)
pixivArtist = PixivModelWhiteCube.PixivArtist(artist_id)
self.getMemberInfoWhitecube(artist_id, pixivArtist)
result.artistName = pixivArtist.artistName
result.artistToken = pixivArtist.artistToken
return result
def cleanUp(self):
try:
print "Start clean-up operation."
print "Selecting all images, this may take some times."
c = self.conn.cursor()
c.execute('''SELECT image_id, save_name from pixiv_master_image''')
print "Checking images."
for row in c:
if not os.path.exists(row[1]):
PixivHelper.safePrint("Missing: " + str(row[0]) + " at " + row[1] + "\n")
self.deleteImage(row[0])
self.conn.commit()
except:
print 'Error at cleanUp():', str(sys.exc_info())
print 'failed'
raise
finally:
c.close()
def ParseWorksData(self, page):
temp = page.find(attrs={'class': 'meta'}).findAll('li')
metaClass = page.find(attrs={'class': 'meta'})
# 07/22/2011 03:09|512×600|RETAS STUDIO
# 07/26/2011 00:30|Manga 39P|ComicStudio 鉛筆 つけペン
# 1/05/2011 07:09|723×1023|Photoshop SAI [ R-18 ]
# 2013年3月16日 06:44 | 800×1130 | Photoshop ComicStudio | R-18
# 2013年12月14日 19:00 855×1133 PhotoshopSAI
self.worksDate = PixivHelper.toUnicode(temp[0].string, encoding=sys.stdin.encoding)
if self.dateFormat is not None and len(self.dateFormat) > 0 and '%' in self.dateFormat:
# use the user defined format
try:
self.worksDateDateTime = datetime.strptime(self.worksDate, self.dateFormat)
except ValueError as ve:
PixivHelper.GetLogger().exception(
'Error when parsing datetime: {0} for imageId {1} using date format {2}'.format(self.worksDate,
str(self.imageId),
str(self.dateFormat)),
ve)
raise
else:
self.worksDate = self.worksDate.replace(u'/', u'-')
if self.worksDate.find('-') > -1:
try:
self.worksDateDateTime = datetime.strptime(self.worksDate, u'%m-%d-%Y %H:%M')
except ValueError as ve:
PixivHelper.GetLogger().exception(
'Error when parsing datetime: {0} for imageId {1}'.format(self.worksDate, self.imageId), ve)
self.worksDateDateTime = datetime.strptime(self.worksDate.split(" ")[0], u'%Y-%m-%d')
else:
tempDate = self.worksDate.replace(u'年', '-').replace(u'月', '-').replace(u'日', '')
self.worksDateDateTime = datetime.strptime(tempDate, '%Y-%m-%d %H:%M')
print "[DEBUG] %s" % self.worksDateDateTime
self.worksResolution = unicode(temp[1].string).replace(u'×', u'x')
self.imageWidth = int(self.worksResolution.split(u'x')[0])
self.imageHeight = int(self.worksResolution.split(u'x')[1])
toolsTemp = page.find(attrs={'class': 'meta'}).find(attrs={'class': 'tools'})
if toolsTemp is not None and len(toolsTemp) > 0:
tools = toolsTemp.findAll('li')
for tool in tools:
# Change delimiter to '^'
if self.worksTools != '':
self.worksTools = self.worksTools + '^' + unicode(tool.string)
else:
self.worksTools = unicode(tool.string)
self.worksTools = self.worksTools.strip()
originalTemp = metaClass.find(attrs={'class': 'original-works _ui-tooltip'})
if originalTemp is not None and len(originalTemp) > 0:
self.isOriginal =True
r18Temp = metaClass.find(attrs={'class': 'r-18'})
if r18Temp is not None and len(r18Temp) > 0:
self.isR18 =True
def WriteXMP(self, filename):
import pyexiv2
# import tempfile
# need to use temp file due to bad unicode support for pyexiv2 in windows
d = PixivHelper.create_temp_dir(prefix="xmp")
tempname = f"{d}/{self.imageId}.xmp"
info = codecs.open(tempname, 'wb', encoding='utf-8')
# Create the XMP file template.
info.write(
'<?xpacket begin="" id=""?>\n<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="XMP Core 4.4.0-Exiv2">\n</x:xmpmeta>\n<?xpacket end="w"?>\n'
)
info.close()
# Reopen file using pyexiv2
# newer version e.g. pyexiv2-2.7.0
info = pyexiv2.Image(tempname)
info_dict = info.read_xmp()
info_dict['Xmp.dc.creator'] = [self.artist.artistName]
# Check array isn't empty.
if self.imageTitle:
info_dict['Xmp.dc.title'] = self.imageTitle
# Check array isn't empty.
if self.imageCaption:
info_dict['Xmp.dc.description'] = self.imageCaption
# Check array isn't empty.
if self.imageTags:
info_dict['Xmp.dc.subject'] = self.imageTags
info_dict['Xmp.dc.date'] = [self.worksDateDateTime]
info_dict[
'Xmp.dc.source'] = f"http://www.pixiv.net/en/artworks/{self.imageId}"
info_dict['Xmp.dc.identifier'] = self.imageId
# Custom 'pixiv' namespace for non-standard details.
pyexiv2.registerNs('http://pixiv.com/', 'pixiv')
info_dict['Xmp.pixiv.artist_id'] = self.artist.artistId
info_dict['Xmp.pixiv.image_mode'] = self.imageMode
info_dict['Xmp.pixiv.pages'] = self.imageCount
info_dict['Xmp.pixiv.resolution'] = self.worksResolution
info_dict['Xmp.pixiv.bookmark_count'] = self.bookmark_count
if self.seriesNavData:
info_dict['Xmp.pixiv.series_title'] = self.seriesNavData['title']
info_dict['Xmp.pixiv.series_order'] = self.seriesNavData['order']
info_dict['Xmp.pixiv.series_id'] = self.seriesNavData['seriesId']
if self.ugoira_data:
info_dict['Xmp.pixiv.ugoira_data'] = self.ugoira_data
if len(self.descriptionUrlList) > 0:
info_dict['Xmp.pixiv.urls'] = ", ".join(self.descriptionUrlList)
# Issue #1064
if len(self.translated_work_title) > 0:
info_dict[
'Xmp.pixiv.translated_work_title'] = self.translated_work_title
if len(self.translated_work_caption) > 0:
info_dict[
'Xmp.pixiv.translated_work_caption'] = self.translated_work_caption
info.modify_xmp(info_dict)
info.close()
# rename to actual file
try:
# Issue #421 ensure subdir exists.
PixivHelper.makeSubdirs(filename)
shutil.move(tempname, filename)
except IOError:
shutil.move(tempname, f"{self.imageId}.xmp")
PixivHelper.get_logger().exception(
"Error when saving image info: %s, file is saved to: %s.xmp",
filename, str(self.imageId))
def PrintInfo(self):
PixivHelper.safePrint('Artist Info')
PixivHelper.safePrint('id : ' + str(self.artistId))
PixivHelper.safePrint('name : ' + self.artistName)
PixivHelper.safePrint('avatar: ' + self.artistAvatar)
PixivHelper.safePrint('token : ' + self.artistToken)
PixivHelper.safePrint('urls : ')
for item in self.imageList:
PixivHelper.safePrint('\t' + str(item))
class TestPixivModel_Fanbox(unittest.TestCase):
currPath = os.path.abspath('.')
PixivHelper.get_logger()
def testFanboxSupportedArtist(self):
p = open('./test/Fanbox_supported_artist.json', 'r').read()
result = Fanbox(p)
self.assertIsNotNone(result)
self.assertEqual(len(result.supportedArtist), 19)
self.assertTrue(7968 in result.supportedArtist)
self.assertTrue(3931854 in result.supportedArtist)
self.assertTrue(30716447 in result.supportedArtist)
def testFanboxArtistPosts(self):
p = open('./test/Fanbox_artist_posts.json', 'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 15521131)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)
for post in result.posts:
self.assertFalse(post.is_restricted)
# post-136761 image
self.assertEqual(result.posts[0].imageId, 136761)
self.assertTrue(len(result.posts[0].imageTitle) > 0)
self.assertTrue(len(result.posts[0].coverImageUrl) > 0)
self.assertEqual(result.posts[0].type, "image")
self.assertEqual(len(result.posts[0].images), 5)
# post-132919 text
self.assertEqual(result.posts[2].imageId, 132919)
self.assertTrue(len(result.posts[2].imageTitle) > 0)
self.assertIsNone(result.posts[2].coverImageUrl)
self.assertEqual(result.posts[2].type, "text")
self.assertEqual(len(result.posts[2].images), 0)
# post-79695 image
self.assertEqual(result.posts[3].imageId, 79695)
self.assertTrue(len(result.posts[3].imageTitle) > 0)
self.assertIsNone(result.posts[3].coverImageUrl)
self.assertEqual(result.posts[3].type, "image")
self.assertEqual(len(result.posts[3].images), 4)
def testFanboxArtistArticle(self):
p = open('./test/Fanbox_artist_posts_article.json', 'r').read()
result = FanboxArtist(190026, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 190026)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)
# post-201946 article
self.assertEqual(result.posts[0].imageId, 201946)
self.assertTrue(len(result.posts[0].imageTitle) > 0)
self.assertIsNone(result.posts[0].coverImageUrl)
self.assertEqual(result.posts[0].type, "article")
self.assertEqual(len(result.posts[0].images), 5)
self.assertEqual(len(result.posts[0].body_text), 1292)
# result.posts[0].WriteInfo("./201946.txt")
def testFanboxArtistArticleFileMap(self):
p = open('./test/creator_with_filemap.json', 'r').read()
result = FanboxArtist(190026, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 190026)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)
# post-201946 article
self.assertEqual(result.posts[0].imageId, 210980)
self.assertTrue(len(result.posts[0].imageTitle) > 0)
self.assertIsNone(result.posts[0].coverImageUrl)
self.assertEqual(result.posts[0].type, "article")
self.assertEqual(len(result.posts[0].images), 15)
self.assertEqual(len(result.posts[0].body_text), 3006)
# result.posts[0].WriteInfo("./210980.txt")
def testFanboxArtistVideo(self):
p = open('./test/creator_posts_with_video.json', 'r').read()
result = FanboxArtist(711048, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 711048)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)
# post-201946 article
self.assertEqual(result.posts[4].imageId, 330905)
self.assertTrue(len(result.posts[4].imageTitle) > 0)
self.assertEqual(
result.posts[4].coverImageUrl,
u'https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/330905/cover/3A2zPUg4s6iz17MM0Z45eWBj.jpeg'
)
self.assertEqual(result.posts[4].type, "video")
self.assertEqual(len(result.posts[4].images), 0)
self.assertEqual(len(result.posts[4].body_text), 99)
# result.posts[0].WriteInfo("./210980.txt")
def testFanboxArtistArticleEmbedTwitter(self):
p = open('./test/creator_embedMap.json', 'r').read()
result = FanboxArtist(68813, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 68813)
self.assertFalse(result.hasNextPage)
self.assertTrue(len(result.posts) > 0)
# post-201946 article
self.assertEqual(result.posts[0].imageId, 285502)
self.assertTrue(len(result.posts[0].imageTitle) > 0)
self.assertEqual(
result.posts[0].coverImageUrl,
u'https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/285502/cover/orx9TCsiPFi5sgDdbvg4zwkX.jpeg'
)
self.assertEqual(result.posts[0].type, "article")
self.assertEqual(len(result.posts[0].images), 7)
self.assertEqual(len(result.posts[0].body_text), 3095)
# result.posts[0].WriteInfo("./285502.txt")
def testFanboxArtistPostsNextPage(self):
p2 = open('./test/Fanbox_artist_posts_nextpage.json', 'r').read()
result = FanboxArtist(15521131, p2)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 15521131)
self.assertFalse(result.hasNextPage)
self.assertTrue(result.nextUrl is None)
self.assertEqual(len(result.posts), 1)
def testFanboxArtistPostsRestricted(self):
p = open('./test/Fanbox_artist_posts_restricted.json', 'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 15521131)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertEqual(len(result.posts), 10)
for post in result.posts:
self.assertTrue(post.is_restricted)
def testFanboxArtistPostsRestrictedNextPage(self):
p = open('./test/Fanbox_artist_posts_next_page_restricted.json',
'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)
self.assertEqual(result.artistId, 15521131)
self.assertFalse(result.hasNextPage)
self.assertTrue(result.nextUrl is None)
self.assertEqual(len(result.posts), 6)
self.assertTrue(result.posts[0].is_restricted)
self.assertFalse(result.posts[1].is_restricted)
def testFanboxOldApi(self):
p = open('./test/fanbox-posts-old-api.json', 'r').read()
result = FanboxArtist(104409, p)
self.assertIsNotNone(result)
self.assertEqual(len(result.posts), 2)
self.assertEqual(result.posts[0].imageId, 916)
self.assertEqual(len(result.posts[0].images), 8)
print(result.posts[0].images)
self.assertEqual(result.posts[1].imageId, 915)
self.assertEqual(len(result.posts[1].images), 1)
print(result.posts[1].images)
def testFanboxNewApi(self):
p = open('./test/fanbox-posts-new-api.json', 'r').read()
result = FanboxArtist(104409, p)
self.assertIsNotNone(result)
self.assertEqual(len(result.posts), 10)
self.assertEqual(result.posts[0].imageId, 577968)
self.assertEqual(len(result.posts[0].images), 2)
self.assertEqual(result.posts[1].imageId, 535518)
self.assertEqual(len(result.posts[1].images), 2)
def testFanboxFilename(self):
p = open('./test/Fanbox_artist_posts.json', 'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)
root_dir = os.path.abspath(os.path.curdir)
post = result.posts[0]
image_url = post.images[0]
current_page = 0
fake_image_url = image_url.replace(
"{0}/".format(post.imageId),
"{0}_p{1}_".format(post.imageId, current_page))
re_page = temp.findall(fake_image_url)
self.assertIsNotNone(re_page)
self.assertEqual(re_page[0], u"0")
def simple_from_images():
filename_format = '%title%_%urlFilename%'
filename = PixivHelper.make_filename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitize_filename(filename, root_dir)
self.assertEqual(
filename, root_dir + os.sep +
u"アスナさん０２_136761_p0_hcXl48iORoJykmrR3zPZEoUu.jpeg")
simple_from_images()
def more_format():
# from images
filename_format = '%member_id%' + os.sep + '%image_id%_p%page_index%_%title%_%urlFilename%_%works_date%'
filename = PixivHelper.make_filename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitize_filename(filename, root_dir)
self.assertEqual(
filename, root_dir + os.sep + u"15521131" + os.sep +
u"136761_p0_アスナさん０２_136761_p0_hcXl48iORoJykmrR3zPZEoUu_2018-08-26 20_28_16.jpeg"
)
more_format()
def cover_more_format():
# https://pixiv.pximg.net/c/1200x630_90_a2_g5/fanbox/public/images/post/96862/cover/6SRpcQwIUuJdeZbhn5q85l9x.jpeg
fake_image_url = post.coverImageUrl.replace(
"{0}/cover/".format(post.imageId), "{0}_".format(post.imageId))
print(fake_image_url)
filename_format = '%member_id%' + os.sep + '%image_id%_%title%_%urlFilename%_%works_date%'
filename = PixivHelper.make_filename(filename_format,
post,
artistInfo=result,
tagsSeparator=" ",
tagsLimit=0,
fileUrl=fake_image_url,
bookmark=None,
searchTags='')
filename = PixivHelper.sanitize_filename(filename, root_dir)
self.assertEqual(
filename, root_dir + os.sep + u"15521131" + os.sep +
u"136761_アスナさん０２_136761_OqhhcslOfbzZpHyTfJNtnIWm_2018-08-26 20_28_16.jpeg"
)
cover_more_format()
def loadConfig(self, path=None):
if path != None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print 'Reading', self.configFileLocation, '...'
oldSetting = False
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication', 'username')
self.password = config.get('Authentication', 'password')
self.cookie = config.get('Authentication', 'cookie')
self.tagsSeparator = PixivHelper.toUnicode(
config.get('Settings', 'tagsseparator'),
encoding=sys.stdin.encoding)
self.rootDirectory = PixivHelper.toUnicode(
config.get('Settings', 'rootdirectory'),
encoding=sys.stdin.encoding)
try:
self.IrfanViewPath = PixivHelper.toUnicode(
config.get('Settings', 'IrfanViewPath'),
encoding=sys.stdin.encoding)
self.downloadListDirectory = PixivHelper.toUnicode(
config.get('Settings', 'downloadListDirectory'),
encoding=sys.stdin.encoding)
except:
pass
try:
self.processFromDb = config.getboolean('Settings',
'processfromdb')
except ValueError:
print "processFromDb = True"
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings',
'daylastupdated')
except ValueError:
print "dayLastUpdated = 7"
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings', 'datediff')
except ValueError:
print "dateDiff = 0"
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Settings', 'proxyaddress')
except ValueError:
print "proxyAddress = ''"
self.proxyAddress = ''
haveError = True
self.proxy = {
'http': self.proxyAddress,
'https': self.proxyAddress
}
try:
self.useProxy = config.getboolean('Settings', 'useproxy')
except ValueError:
print "useProxy = False"
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings', 'uselist')
except ValueError:
print "useList = False"
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv', 'r18mode')
except ValueError:
print "r18mode = False"
self.r18mode = False
haveError = True
_useragent = config.get('Settings', 'useragent')
if _useragent != None:
self.useragent = _useragent
_filenameFormat = config.get('Settings', 'filenameformat')
_filenameFormat = PixivHelper.toUnicode(
_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat != None:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Settings',
'filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(
_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat != None:
## check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find(
'%page_index%'
) == -1 and _filenameMangaFormat.find(
'%page_number%') == -1:
print 'No page identifier, appending %page_index% to the filename manga format.'
_filenameMangaFormat = _filenameMangaFormat + unicode(
' %page_index%')
print "_filenameMangaFormat =", _filenameMangaFormat
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
try:
self.debugHttp = config.getboolean('Settings', 'debughttp')
except ValueError:
self.debugHttp = False
print "debugHttp = False"
haveError = True
try:
self.useRobots = config.getboolean('Settings', 'userobots')
except ValueError:
self.useRobots = False
print "useRobots = False"
haveError = True
try:
self.overwrite = config.getboolean('Settings', 'overwrite')
except ValueError:
print "overwrite = False"
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean(
'Settings', 'createMangaDir')
except ValueError:
print "createMangaDir = False"
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Settings', 'timeout')
except ValueError:
print "timeout = 60"
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Settings', 'retry')
except ValueError:
print "retry = 3"
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Settings', 'retrywait')
except ValueError:
print "retryWait = 5"
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv', 'numberofpage')
except ValueError:
self.numberOfPage = 0
print "numberOfPage = 0"
haveError = True
try:
self.createDownloadLists = config.getboolean(
'Settings', 'createDownloadLists')
except ValueError:
self.createDownloadLists = False
print "createDownloadLists = False"
haveError = True
try:
self.startIrfanView = config.getboolean(
'Settings', 'startIrfanView')
except ValueError:
self.startIrfanView = False
print "startIrfanView = False"
haveError = True
try:
self.startIrfanSlide = config.getboolean(
'Settings', 'startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print "startIrfanSlide = False"
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean(
'Settings', 'alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print "alwaysCheckFileSize = False"
haveError = True
try:
self.downloadAvatar = config.getboolean(
'Settings', 'downloadAvatar')
except ValueError:
self.downloadAvatar = False
print "downloadAvatar = False"
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings',
'checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print "checkUpdatedLimit = 0"
haveError = True
try:
self.useTagsAsDir = config.getboolean('Settings',
'useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print "useTagsAsDir = False"
haveError = True
try:
self.useBlacklistTags = config.getboolean(
'Settings', 'useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print "useBlacklistTags = False"
haveError = True
try:
self.useSuppressTags = config.getboolean(
'Settings', 'useSuppressTags')
except ValueError:
self.useSuppressTags = False
print "useSuppressTags = False"
haveError = True
try:
self.tagsLimit = config.getint('Settings', 'tagsLimit')
except ValueError:
self.tagsLimit = -1
print "tagsLimit = -1"
haveError = True
try:
self.useSSL = config.getboolean('Authentication', 'useSSL')
except ValueError:
self.useSSL = False
print "useSSL = False"
haveError = True
try:
self.writeImageInfo = config.getboolean(
'Settings', 'writeImageInfo')
except ValueError:
self.writeImageInfo = False
print "writeImageInfo = False"
haveError = True
try:
self.keepSignedIn = config.getint('Authentication',
'keepSignedIn')
except ValueError:
print "keepSignedIn = 0"
self.keepSignedIn = 0
haveError = True
try:
self.backupOldFile = config.getboolean('Settings',
'backupOldFile')
except ValueError:
self.backupOldFile = False
print "backupOldFile = False"
haveError = True
try:
self.logLevel = config.get('Settings', 'logLevel').upper()
if not self.logLevel in [
'CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO',
'DEBUG', 'NOTSET'
]:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print "logLevel = DEBUG"
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Settings', 'enableDump')
except ValueError:
print "enableDump = True"
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Settings', 'skipDumpFilter')
except ValueError:
print "skipDumpFilter = ''"
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean(
'Settings', 'dumpMediumPage')
except ValueError:
print "dumpMediumPage = False"
self.dumpMediumPage = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean(
'Settings', 'writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print "writeUgoiraInfo = False"
haveError = True
try:
self.createUgoira = config.getboolean('Settings',
'createUgoira')
except ValueError:
self.createUgoira = False
print "createUgoira = False"
haveError = True
try:
self.deleteZipFile = config.getboolean('Settings',
'deleteZipFile')
except ValueError:
self.deleteZipFile = False
print "deleteZipFile = False"
haveError = True
## except ConfigParser.NoOptionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
## except ConfigParser.NoSectionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
except:
print 'Error at loadConfig():', sys.exc_info()
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print 'Some configuration have invalid value, replacing with the default value.'
self.writeConfig(error=True)
print 'done.'
class PixivConfig(object):
'''Configuration class'''
__logger = PixivHelper.GetLogger()
configFileLocation = "config.ini"
# initialize default value
# Network related
proxyAddress = ''
proxy = {
'http': proxyAddress,
'https': proxyAddress,
}
useProxy = False
useragent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36'
useRobots = True
timeout = 60
retry = 3
retryWait = 5
downloadDelay = 2
checkNewVersion = True
# Authentication related
username = ''
password = ''
cookie = ''
# Pixiv related?
numberOfPage = 0
r18mode = False
dateFormat = ''
# generic Settings
filenameFormat = unicode('%artist% (%member_id%)' + os.sep +
'%urlFilename% - %title%')
filenameMangaFormat = unicode('%artist% (%member_id%)' + os.sep +
'%urlFilename% - %title%')
filenameInfoFormat = unicode('%artist% (%member_id%)' + os.sep +
'%urlFilename% - %title%')
rootDirectory = unicode('.')
overwrite = False
useList = False
processFromDb = True
dayLastUpdated = 7
tagsSeparator = unicode(', ')
alwaysCheckFileSize = False
checkUpdatedLimit = 0
downloadAvatar = True
createMangaDir = False
useTagsAsDir = False
useBlacklistTags = False
useSuppressTags = False
tagsLimit = -1
writeImageInfo = False
writeImageJSON = False
dateDiff = 0
backupOldFile = False
enableInfiniteLoop = False
verifyImage = False
writeUrlInDescription = False
urlBlacklistRegex = ""
urlDumpFilename = "url_list_%Y%m%d"
dbPath = ''
useBlacklistMembers = False
avatarNameFormat = ""
setLastModified = True
alwaysCheckFileExists = False
useLocalTimezone = False # Issue #420
# ugoira
writeUgoiraInfo = False
createUgoira = False
deleteZipFile = False
createGif = False
createApng = False
deleteUgoira = False
createWebm = False
createWebp = False
# IrfanView
createDownloadLists = False
downloadListDirectory = unicode('.')
startIrfanView = False
startIrfanSlide = False
IrfanViewPath = unicode(r'C:\Program Files\IrfanView')
# FFmpeg
ffmpeg = "ffmpeg"
ffmpegCodec = "libvpx-vp9"
ffmpegParam = "-lossless 1 -vsync 2 -r 999 -pix_fmt yuv420p"
webpCodec = "libwebp"
webpParam = "-lossless 0 -q:v 90 -loop 0 -vsync 2 -r 999"
# Debug related
logLevel = "DEBUG"
enableDump = True
skipDumpFilter = ""
dumpMediumPage = False
dumpTagSearchPage = False
debugHttp = False
def loadConfig(self, path=None):
''' new settings must be added on the last'''
if path is not None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print('Reading', self.configFileLocation, '...')
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication', 'username')
self.password = config.get('Authentication', 'password')
self.cookie = config.get('Authentication', 'cookie')
self.tagsSeparator = PixivHelper.toUnicode(
config.get('Settings', 'tagsseparator'),
encoding=sys.stdin.encoding)
self.rootDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings', 'rootdirectory'),
encoding=sys.stdin.encoding))
try:
self.IrfanViewPath = os.path.expanduser(
PixivHelper.toUnicode(config.get('IrfanView',
'IrfanViewPath'),
encoding=sys.stdin.encoding))
self.downloadListDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings',
'downloadListDirectory'),
encoding=sys.stdin.encoding))
except BaseException:
pass
try:
self.processFromDb = config.getboolean('Settings',
'processfromdb')
except ValueError:
print("processFromDb = True")
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings',
'daylastupdated')
except ValueError:
print("dayLastUpdated = 7")
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings', 'datediff')
except ValueError:
print("dateDiff = 0")
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Network', 'proxyaddress')
except ValueError:
print("proxyAddress = ''")
self.proxyAddress = ''
haveError = True
self.proxy = {
'http': self.proxyAddress,
'https': self.proxyAddress
}
try:
self.useProxy = config.getboolean('Network', 'useproxy')
except ValueError:
print("useProxy = False")
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings', 'uselist')
except ValueError:
print("useList = False")
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv', 'r18mode')
except ValueError:
print("r18mode = False")
self.r18mode = False
haveError = True
_useragent = config.get('Network', 'useragent')
if _useragent is not None:
self.useragent = _useragent
_filenameFormat = config.get('Settings', 'filenameformat')
_filenameFormat = PixivHelper.toUnicode(
_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat is not None and len(_filenameFormat) > 0:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Settings',
'filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(
_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat is not None and len(
_filenameMangaFormat) > 0:
# check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find(
'%page_index%'
) == -1 and _filenameMangaFormat.find(
'%page_number%') == -1:
print(
'No page identifier, appending %page_index% to the filename manga format.'
)
_filenameMangaFormat = _filenameMangaFormat + unicode(
' %page_index%')
print("_filenameMangaFormat =", _filenameMangaFormat)
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
_filenameInfoFormat = config.get('Settings', 'filenameinfoformat')
_filenameInfoFormat = PixivHelper.toUnicode(
_filenameInfoFormat, encoding=sys.stdin.encoding)
if _filenameInfoFormat is not None and len(
_filenameInfoFormat) > 0:
self.filenameInfoFormat = _filenameInfoFormat
try:
self.debugHttp = config.getboolean('Debug', 'debughttp')
except ValueError:
self.debugHttp = False
print("debugHttp = False")
haveError = True
try:
self.useRobots = config.getboolean('Network', 'userobots')
except ValueError:
self.useRobots = False
print("useRobots = False")
haveError = True
try:
self.overwrite = config.getboolean('Settings', 'overwrite')
except ValueError:
print("overwrite = False")
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean(
'Settings', 'createMangaDir')
except ValueError:
print("createMangaDir = False")
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Network', 'timeout')
except ValueError:
print("timeout = 60")
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Network', 'retry')
except ValueError:
print("retry = 3")
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Network', 'retrywait')
except ValueError:
print("retryWait = 5")
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv', 'numberofpage')
except ValueError:
self.numberOfPage = 0
print("numberOfPage = 0")
haveError = True
try:
self.createDownloadLists = config.getboolean(
'IrfanView', 'createDownloadLists')
except ValueError:
self.createDownloadLists = False
print("createDownloadLists = False")
haveError = True
try:
self.startIrfanView = config.getboolean(
'IrfanView', 'startIrfanView')
except ValueError:
self.startIrfanView = False
print("startIrfanView = False")
haveError = True
try:
self.startIrfanSlide = config.getboolean(
'IrfanView', 'startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print("startIrfanSlide = False")
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean(
'Settings', 'alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print("alwaysCheckFileSize = False")
haveError = True
try:
self.downloadAvatar = config.getboolean(
'Settings', 'downloadAvatar')
except ValueError:
self.downloadAvatar = False
print("downloadAvatar = False")
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings',
'checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print("checkUpdatedLimit = 0")
haveError = True
try:
self.useTagsAsDir = config.getboolean('Settings',
'useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print("useTagsAsDir = False")
haveError = True
try:
self.useBlacklistTags = config.getboolean(
'Settings', 'useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print("useBlacklistTags = False")
haveError = True
try:
self.useSuppressTags = config.getboolean(
'Settings', 'useSuppressTags')
except ValueError:
self.useSuppressTags = False
print("useSuppressTags = False")
haveError = True
try:
self.tagsLimit = config.getint('Settings', 'tagsLimit')
except ValueError:
self.tagsLimit = -1
print("tagsLimit = -1")
haveError = True
try:
self.writeImageInfo = config.getboolean(
'Settings', 'writeImageInfo')
except ValueError:
self.writeImageInfo = False
print("writeImageInfo = False")
haveError = True
try:
self.backupOldFile = config.getboolean('Settings',
'backupOldFile')
except ValueError:
self.backupOldFile = False
print("backupOldFile = False")
haveError = True
try:
self.logLevel = config.get('Debug', 'logLevel').upper()
if self.logLevel not in [
'CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO',
'DEBUG', 'NOTSET'
]:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print("logLevel = DEBUG")
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Debug', 'enableDump')
except ValueError:
print("enableDump = True")
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Debug', 'skipDumpFilter')
except ValueError:
print("skipDumpFilter = ''")
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean(
'Debug', 'dumpMediumPage')
except ValueError:
print("dumpMediumPage = False")
self.dumpMediumPage = False
haveError = True
try:
self.enableInfiniteLoop = config.getboolean(
'Settings', 'enableInfiniteLoop')
except ValueError:
self.enableInfiniteLoop = False
print("enableInfiniteLoop = False")
haveError = True
try:
self.dumpTagSearchPage = config.getboolean(
'Debug', 'dumpTagSearchPage')
except ValueError:
self.dumpTagSearchPage = False
print("dumpTagSearchPage = False")
haveError = True
try:
self.dateFormat = config.get('Pixiv', 'dateFormat')
except ValueError:
print("dateFormat = ''")
self.dateFormat = ''
haveError = True
try:
self.verifyImage = config.getboolean('Settings', 'verifyImage')
except ValueError:
print("verifyImage = False")
self.verifyImage = False
haveError = True
try:
self.writeUrlInDescription = config.getboolean(
'Settings', 'writeUrlInDescription')
except ValueError:
print("writeUrlInDescription = False")
self.writeUrlInDescription = False
haveError = True
try:
self.urlBlacklistRegex = config.get('Settings',
'urlBlacklistRegex')
except ValueError:
print("urlBlacklistRegex = ")
self.urlBlacklistRegex = ""
haveError = True
try:
self.urlDumpFilename = config.get('Settings',
'urlDumpFilename')
except ValueError:
print("urlDumpFilename = url_list_%Y%m%d")
self.urlDumpFilename = "url_list_%Y%m%d"
haveError = True
try:
self.dbPath = config.get('Settings', 'dbPath')
except ValueError:
print("dbPath = ''")
self.dbPath = ''
haveError = True
try:
self.useBlacklistMembers = config.getboolean(
'Settings', 'useBlacklistMembers')
except ValueError:
print("useBlacklistMembers = False")
self.useBlacklistMembers = False
haveError = True
try:
self.avatarNameFormat = config.get('Settings',
'avatarNameFormat')
self.avatarNameFormat = PixivHelper.toUnicode(
self.avatarNameFormat, encoding=sys.stdin.encoding)
except ValueError:
print("avatarNameFormat = ")
self.avatarNameFormat = ""
haveError = True
try:
self.writeImageJSON = config.getboolean(
'Settings', 'writeImageJSON')
except ValueError:
self.writeImageJSON = False
print("writeImageJSON = False")
haveError = True
try:
self.downloadDelay = config.getint('Network', 'downloadDelay')
except ValueError:
self.downloadDelay = 2
print("downloadDelay = 2")
haveError = True
try:
self.ffmpeg = config.get('FFmpeg', 'ffmpeg')
except ValueError:
print("ffmpeg = 'ffmpeg'")
self.ffmpeg = 'ffmpeg'
haveError = True
try:
self.ffmpegCodec = config.get('FFmpeg', 'ffmpegCodec')
except ValueError:
print("ffmpegCodec = 'libvpx-vp9'")
self.ffmpegCodec = 'libvpx-vp9'
haveError = True
try:
self.ffmpegParam = config.get('FFmpeg', 'ffmpegParam')
except ValueError:
print("ffmpegParam = '-lossless 1'")
self.ffmpegParam = '-lossless 1'
haveError = True
try:
self.setLastModified = config.getboolean(
'Settings', 'setLastModified')
except ValueError:
print("setLastModified = True")
self.setLastModified = True
haveError = True
try:
self.useLocalTimezone = config.getboolean(
'Settings', 'useLocalTimezone')
except ValueError:
print("useLocalTimezone = False")
self.useLocalTimezone = False
haveError = True
try:
self.checkNewVersion = config.getboolean(
'Network', 'checkNewVersion')
except ValueError:
print("checkNewVersion = True")
self.checkNewVersion = True
haveError = True
try:
self.webpCodec = config.get('FFmpeg', 'webpCodec')
except ValueError:
print("webpCodec = 'libwebp'")
self.webpCodec = 'libwebp'
haveError = True
try:
self.webpParam = config.get('FFmpeg', 'webpParam')
except ValueError:
print("webpParam = '-lossless 0 -q:v 90'")
self.webpParam = '-lossless 0 -q:v 90'
haveError = True
try:
self.createWebp = config.getboolean('Ugoira', 'createWebp')
except ValueError:
print("createWebp = False")
self.createWebp = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean(
'Ugoira', 'writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print("writeUgoiraInfo = False")
haveError = True
try:
self.createUgoira = config.getboolean('Ugoira', 'createUgoira')
except ValueError:
self.createUgoira = False
print("createUgoira = False")
haveError = True
try:
self.deleteZipFile = config.getboolean('Ugoira',
'deleteZipFile')
except ValueError:
self.deleteZipFile = False
print("deleteZipFile = False")
haveError = True
try:
self.createGif = config.getboolean('Ugoira', 'createGif')
except ValueError:
print("createGif = False")
self.createGif = False
haveError = True
try:
self.createApng = config.getboolean('Ugoira', 'createApng')
except ValueError:
print("createApng = False")
self.createApng = False
haveError = True
try:
self.deleteUgoira = config.getboolean('Ugoira', 'deleteUgoira')
except ValueError:
print("deleteUgoira = False")
self.deleteUgoira = False
haveError = True
try:
self.createWebm = config.getboolean('Ugoira', 'createWebm')
except ValueError:
print("createWebm = False")
self.createWebm = False
haveError = True
except BaseException:
print('Error at loadConfig():', sys.exc_info())
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print(
'Some configuration have invalid value, replacing with the default value.'
)
self.writeConfig(error=True)
print('done.')
# -UI01B------write config
def writeConfig(self, error=False, path=None):
'''Backup old config if exist and write updated config.ini'''
print('Writing config file...', end=' ')
config = ConfigParser.RawConfigParser()
config.add_section('Network')
config.set('Network', 'useProxy', self.useProxy)
config.set('Network', 'proxyAddress', self.proxyAddress)
config.set('Network', 'useragent', self.useragent)
config.set('Network', 'useRobots', self.useRobots)
config.set('Network', 'timeout', self.timeout)
config.set('Network', 'retry', self.retry)
config.set('Network', 'retrywait', self.retryWait)
config.set('Network', 'downloadDelay', self.downloadDelay)
config.set('Network', 'checkNewVersion', self.checkNewVersion)
config.add_section('Debug')
config.set('Debug', 'logLevel', self.logLevel)
config.set('Debug', 'enableDump', self.enableDump)
config.set('Debug', 'skipDumpFilter', self.skipDumpFilter)
config.set('Debug', 'dumpMediumPage', self.dumpMediumPage)
config.set('Debug', 'dumpTagSearchPage', self.dumpTagSearchPage)
config.set('Debug', 'debugHttp', self.debugHttp)
config.add_section('IrfanView')
config.set('IrfanView', 'IrfanViewPath', self.IrfanViewPath)
config.set('IrfanView', 'startIrfanView', self.startIrfanView)
config.set('IrfanView', 'startIrfanSlide', self.startIrfanSlide)
config.set('IrfanView', 'createDownloadLists',
self.createDownloadLists)
config.add_section('Settings')
config.set('Settings', 'filenameFormat', self.filenameFormat)
config.set('Settings', 'filenameMangaFormat', self.filenameMangaFormat)
config.set('Settings', 'filenameInfoFormat', self.filenameInfoFormat)
config.set('Settings', 'avatarNameFormat', self.avatarNameFormat)
config.set('Settings', 'downloadListDirectory',
self.downloadListDirectory)
config.set('Settings', 'useList', self.useList)
config.set('Settings', 'processFromDb', self.processFromDb)
config.set('Settings', 'overwrite', self.overwrite)
config.set('Settings', 'tagsseparator', self.tagsSeparator)
config.set('Settings', 'daylastupdated', self.dayLastUpdated)
config.set('Settings', 'rootdirectory', self.rootDirectory)
config.set('Settings', 'alwaysCheckFileSize', self.alwaysCheckFileSize)
config.set('Settings', 'checkUpdatedLimit', self.checkUpdatedLimit)
config.set('Settings', 'downloadAvatar', self.downloadAvatar)
config.set('Settings', 'createMangaDir', self.createMangaDir)
config.set('Settings', 'useTagsAsDir', self.useTagsAsDir)
config.set('Settings', 'useBlacklistTags', self.useBlacklistTags)
config.set('Settings', 'useSuppressTags', self.useSuppressTags)
config.set('Settings', 'tagsLimit', self.tagsLimit)
config.set('Settings', 'writeImageInfo', self.writeImageInfo)
config.set('Settings', 'writeImageJSON', self.writeImageJSON)
config.set('Settings', 'dateDiff', self.dateDiff)
config.set('Settings', 'backupOldFile', self.backupOldFile)
config.set('Settings', 'enableInfiniteLoop', self.enableInfiniteLoop)
config.set('Settings', 'verifyImage', self.verifyImage)
config.set('Settings', 'writeUrlInDescription',
self.writeUrlInDescription)
config.set('Settings', 'urlBlacklistRegex', self.urlBlacklistRegex)
config.set('Settings', 'urlDumpFilename', self.urlDumpFilename)
config.set('Settings', 'dbPath', self.dbPath)
config.set('Settings', 'useBlacklistMembers', self.useBlacklistMembers)
config.set('Settings', 'setLastModified', self.setLastModified)
config.set('Settings', 'useLocalTimezone', self.useLocalTimezone)
config.add_section('Authentication')
config.set('Authentication', 'username', self.username)
config.set('Authentication', 'password', self.password)
config.set('Authentication', 'cookie', self.cookie)
config.add_section('Pixiv')
config.set('Pixiv', 'numberOfPage', self.numberOfPage)
config.set('Pixiv', 'R18Mode', self.r18mode)
config.set('Pixiv', 'DateFormat', self.dateFormat)
config.add_section('FFmpeg')
config.set('FFmpeg', 'ffmpeg', self.ffmpeg)
config.set('FFmpeg', 'ffmpegCodec', self.ffmpegCodec)
config.set('FFmpeg', 'ffmpegParam', self.ffmpegParam)
config.set('FFmpeg', 'webpCodec', self.webpCodec)
config.set('FFmpeg', 'webpParam', self.webpParam)
config.add_section('Ugoira')
config.set('Ugoira', 'writeUgoiraInfo', self.writeUgoiraInfo)
config.set('Ugoira', 'createUgoira', self.createUgoira)
config.set('Ugoira', 'deleteZipFile', self.deleteZipFile)
config.set('Ugoira', 'createGif', self.createGif)
config.set('Ugoira', 'createApng', self.createApng)
config.set('Ugoira', 'deleteUgoira', self.deleteUgoira)
config.set('Ugoira', 'createWebm', self.createWebm)
config.set('Ugoira', 'createWebp', self.createWebp)
if path is not None:
configlocation = path
else:
configlocation = 'config.ini'
try:
# with codecs.open('config.ini.bak', encoding = 'utf-8', mode = 'wb') as configfile:
with open(configlocation + '.tmp', 'w') as configfile:
config.write(configfile)
if os.path.exists(configlocation):
if error:
backupName = configlocation + '.error-' + str(
int(time.time()))
print("Backing up old config (error exist!) to " +
backupName)
shutil.move(configlocation, backupName)
else:
print("Backing up old config to config.ini.bak")
shutil.move(configlocation, configlocation + '.bak')
os.rename(configlocation + '.tmp', configlocation)
except BaseException:
self.__logger.exception('Error at writeConfig()')
raise
print('done.')
def printConfig(self):
print('Configuration: ')
print(' [Authentication]')
print(' - username ='******' - password = '******' - cookie = ', self.cookie)
print(' [Network]')
print(' - useproxy =', self.useProxy)
print(' - proxyaddress =', self.proxyAddress)
print(' - useragent =', self.useragent)
print(' - use_robots =', self.useRobots)
print(' - timeout =', self.timeout)
print(' - retry =', self.retry)
print(' - retryWait =', self.retryWait)
print(' - downloadDelay =', self.downloadDelay)
print(' - checkNewVersion =', self.checkNewVersion)
print(' [Debug]')
print(' - logLevel =', self.logLevel)
print(' - enableDump =', self.enableDump)
print(' - skipDumpFilter =', self.skipDumpFilter)
print(' - dumpMediumPage =', self.dumpMediumPage)
print(' - dumpTagSearchPage=', self.dumpTagSearchPage)
print(' - debug_http =', self.debugHttp)
print(' [IrfanView]')
print(' - IrfanViewPath =', self.IrfanViewPath)
print(' - startIrfanView =', self.startIrfanView)
print(' - startIrfanSlide =', self.startIrfanSlide)
print(' - createDownloadLists =', self.createDownloadLists)
print(' [Settings]')
print(' - filename_format =', self.filenameFormat)
print(' - filename_manga_format =', self.filenameMangaFormat)
print(' - filename_info_format =', self.filenameInfoFormat)
print(' - avatarNameFormat =', self.avatarNameFormat)
print(' - downloadListDirectory =', self.downloadListDirectory)
print(' - overwrite =', self.overwrite)
print(' - useList =', self.useList)
print(' - processFromDb =', self.processFromDb)
print(' - tagsSeparator =', self.tagsSeparator)
print(' - dayLastUpdated =', self.dayLastUpdated)
print(' - rootDirectory =', self.rootDirectory)
print(' - alwaysCheckFileSize =', self.alwaysCheckFileSize)
print(' - checkUpdatedLimit =', self.checkUpdatedLimit)
print(' - downloadAvatar =', self.downloadAvatar)
print(' - createMangaDir =', self.createMangaDir)
print(' - useTagsAsDir =', self.useTagsAsDir)
print(' - useBlacklistTags =', self.useBlacklistTags)
print(' - useSuppressTags =', self.useSuppressTags)
print(' - tagsLimit =', self.tagsLimit)
print(' - writeImageInfo =', self.writeImageInfo)
print(' - writeImageJSON =', self.writeImageJSON)
print(' - dateDiff =', self.dateDiff)
print(' - backupOldFile =', self.backupOldFile)
print(' - enableInfiniteLoop =', self.enableInfiniteLoop)
print(' - verifyImage =', self.verifyImage)
print(' - writeUrlInDescription =', self.writeUrlInDescription)
print(' - urlBlacklistRegex =', self.urlBlacklistRegex)
print(' - urlDumpFilename =', self.urlDumpFilename)
print(' - dbPath =', self.dbPath)
print(' - useBlacklistMembers =', self.useBlacklistMembers)
print(' - setLastModified =', self.setLastModified)
print(' - useLocalTimezone =', self.useLocalTimezone)
print(' [Pixiv]')
print(' - numberOfPage =', self.numberOfPage)
print(' - R18Mode =', self.r18mode)
print(' - DateFormat =', self.dateFormat)
print(' [FFmpeg]')
print(' - ffmpeg =', self.ffmpeg)
print(' - ffmpegCodec =', self.ffmpegCodec)
print(' - ffmpegParam =', self.ffmpegParam)
print(' - webpCodec =', self.webpCodec)
print(' - webpParam =', self.webpParam)
print(' [Ugoira]')
print(' - writeUgoiraInfo =', self.writeUgoiraInfo)
print(' - createUgoira =', self.createUgoira)
print(' - deleteZipFile =', self.deleteZipFile)
print(' - createGif =', self.createGif)
print(' - createApng =', self.createApng)
print(' - deleteUgoira =', self.deleteUgoira)
print(' - createWebm =', self.createWebm)
print(' - createWebp =', self.createWebp)
print('')
def loadConfig(self, path=None):
if path is not None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print('Reading', self.configFileLocation, '...')
config = configparser.RawConfigParser()
try:
with PixivHelper.open_text_file(self.configFileLocation) as reader:
content = reader.read()
except BaseException:
print('Error at loadConfig() reading file:', self.configFileLocation, "\n", sys.exc_info())
self.__logger.exception('Error at loadConfig() reading file: %s', self.configFileLocation)
self.writeConfig(error=True, path=self.configFileLocation)
return
haveError = False
config.read_string(content)
for item in PixivConfig.__items:
option_type = type(item.default)
method = config.get
if option_type == int:
method = config.getint
elif option_type == bool:
method = config.getboolean
value = None
try:
try:
value = method(item.section, item.option)
except (configparser.NoSectionError, configparser.NoOptionError):
haveError = True
for section in config.sections():
try:
value = method(section, item.option)
break
except (configparser.NoSectionError, configparser.NoOptionError):
continue
if value is None:
raise
except BaseException:
print(item.option, "=", item.default)
value = item.default
haveError = True
# Issue #743
try:
value = item.process_value(value)
except ValueError:
print(Fore.RED + Style.BRIGHT + f"{sys.exc_info()}" + Style.RESET_ALL)
self.__logger.exception('Error at process_value() of : %s', item.option)
print(Fore.YELLOW + Style.BRIGHT + f"{item.option} = {item.default}" + Style.RESET_ALL)
value = item.default
haveError = True
# assign the value to the actual configuration attribute
self.__setattr__(item.option, value)
self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress}
if haveError:
print(Fore.RED + Style.BRIGHT + 'Configurations with invalid value are set to default value.' + Style.RESET_ALL)
self.writeConfig(error=True, path=self.configFileLocation)
print('Configuration loaded.')
def parseBody(self, jsPost):
''' Parse general data for text and article'''
self.body_text = ""
embedData = list()
if "text" in jsPost["body"]:
self.body_text = jsPost["body"]["text"]
# Issue #544
elif "html" in jsPost["body"]:
self.body_text = jsPost["body"]["html"]
# Issue #611: try to parse all images in the html body for compatibility
parsed = BeautifulSoup(self.body_text, features="html5lib")
links = parsed.findAll('a')
for link in links:
if link["href"].find("//fanbox.pixiv.net/images/entry/") > 0:
self.try_add(link["href"], self.embeddedFiles)
self.try_add(link["href"], self.images)
images = parsed.findAll('img')
for image in images:
if "data-src-original" in image.attrs:
self.try_add(image["data-src-original"],
self.embeddedFiles)
self.try_add(image["data-src-original"], self.images)
parsed.decompose()
del parsed
if "thumbnailUrl" in jsPost["body"] and jsPost["body"][
"thumbnailUrl"] is not None:
# set the thumbnail as the cover image is not exists.
if self.coverImageUrl is None:
PixivHelper.get_logger().debug(
"Missing coverImageUrl, using thumbnailUrl instead as cover."
)
self.coverImageUrl = jsPost["body"]["thumbnailUrl"]
self.embeddedFiles.append(jsPost["body"]["thumbnailUrl"])
if "embedMap" in jsPost["body"] and jsPost["body"][
"embedMap"] is not None and len(
jsPost["body"]["embedMap"]) > 0:
for embed in jsPost["body"]["embedMap"]:
embedData.append(jsPost["body"]["embedMap"][embed])
self.embeddedFiles.append(jsPost["body"]["embedMap"][embed])
if "blocks" in jsPost["body"] and jsPost["body"]["blocks"] is not None:
for block in jsPost["body"]["blocks"]:
if block["type"] == "p":
if "links" in block:
pointer = 0
block_text = ""
for i in range(0, len(block["links"])):
link = block["links"][i]
link_offset = link["offset"]
block_text += block["text"][pointer:link_offset]
pointer = link_offset + link["length"]
block_text += "<a href='{0}'>{1}</a>".format(
link["url"],
block["text"][link_offset:pointer])
block_text += block["text"][pointer:]
else:
block_text = block["text"]
self.body_text = u"{0}<p>{1}</p>".format(
self.body_text, block_text)
elif block["type"] == "image":
imageId = block["imageId"]
if imageId not in jsPost["body"]["imageMap"]:
continue
self.body_text = u"{0}<br /><a href='{1}'><img src='{2}'/></a>".format(
self.body_text,
jsPost["body"]["imageMap"][imageId]["originalUrl"],
jsPost["body"]["imageMap"][imageId]["thumbnailUrl"])
self.try_add(
jsPost["body"]["imageMap"][imageId]["originalUrl"],
self.images)
self.try_add(
jsPost["body"]["imageMap"][imageId]["originalUrl"],
self.embeddedFiles)
elif block["type"] == "file":
fileId = block["fileId"]
if fileId not in jsPost["body"]["fileMap"]:
continue
self.body_text = u"{0}<br /><a href='{1}'>{2}</a>".format(
self.body_text,
jsPost["body"]["fileMap"][fileId]["url"],
jsPost["body"]["fileMap"][fileId]["name"])
self.try_add(jsPost["body"]["fileMap"][fileId]["url"],
self.images)
self.try_add(jsPost["body"]["fileMap"][fileId]["url"],
self.embeddedFiles)
elif block["type"] == "embed": # Implement #470
embedId = block["embedId"]
self.body_text = u"{0}<br />{1}".format(
self.body_text,
self.getEmbedData(jsPost["body"]["embedMap"][embedId],
jsPost))
# Issue #476
if "video" in jsPost["body"]:
self.body_text = u"{0}<br />{1}".format(
self.body_text,
self.getEmbedData(jsPost["body"]["video"], jsPost))
def PrintInfo(self):
PixivHelper.safePrint('Artist Info')
PixivHelper.safePrint('id : ' + str(self.artistId))
PixivHelper.safePrint('name : ' + self.artistName)
PixivHelper.safePrint('avatar: ' + self.artistAvatar)
PixivHelper.safePrint('token : ' + self.artistToken)
PixivHelper.safePrint('urls : {0}'.format(len(self.imageList)))
for item in self.imageList:
PixivHelper.safePrint('\t' + str(item))
PixivHelper.safePrint('total : {0}'.format(self.totalImages))
PixivHelper.safePrint('last? : {0}'.format(self.isLastPage))
def IsUserSuspended(self, page):
errorMessages = [
'該当ユーザーのアカウントは停止されています。', 'This user account has been suspended.'
]
return PixivHelper.HaveStrings(page, errorMessages)
def IsNeedAppropriateLevel(self, page):
errorMessages = ['該当作品の公開レベルにより閲覧できません。']
return PixivHelper.HaveStrings(page, errorMessages)
def IsGuroDisabled(self, page):
errorMessages = [
'表示されるページには、18歳未満の方には不適切な表現内容が含まれています。',
'The page you are trying to access contains content that may be unsuitable for minors'
]
return PixivHelper.HaveStrings(page, errorMessages)
def PrintInfo(self):
PixivHelper.safePrint('Image Info')
PixivHelper.safePrint('img id: ' + str(self.imageId))
PixivHelper.safePrint('title : ' + self.imageTitle)
PixivHelper.safePrint('caption : ' + self.imageCaption)
PixivHelper.safePrint('mode : ' + self.imageMode)
PixivHelper.safePrint('tags :', newline=False)
PixivHelper.safePrint(', '.join(self.imageTags))
PixivHelper.safePrint('views : ' + str(self.jd_rtv))
PixivHelper.safePrint('rating: ' + str(self.jd_rtc))
PixivHelper.safePrint('total : ' + str(self.jd_rtt))
PixivHelper.safePrint('Date : ' + self.worksDate)
PixivHelper.safePrint('Resolution : ' + self.worksResolution)
PixivHelper.safePrint('Tools : ' + self.worksTools)
return ""
def process_sketch_artists(caller,
config,
artist_id,
start_page=0,
end_page=0):
config.loadConfig(path=caller.configfile)
br = PixivBrowserFactory.getBrowser()
title_prefix = f"Pixiv Sketch - Processing Artist Id: {artist_id}"
caller.set_console_title(title_prefix)
msg = Fore.YELLOW + Style.NORMAL + f'Processing Artist Id: {artist_id}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
try:
artist = br.sketch_get_posts_by_artist_id(artist_id, end_page)
POST_PER_PAGE = 10
start_idx = POST_PER_PAGE * (start_page - 1)
end_idx = POST_PER_PAGE * (end_page)
if end_page == 0 or end_idx > len(artist.posts):
end_idx = len(artist.posts)
msg = Fore.YELLOW + Style.NORMAL + f'Processing from post #{start_idx} to #{end_idx}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
post_to_process = artist.posts[start_idx:end_idx]
current_post = 1
for item in post_to_process:
caller.set_console_title(
f"{title_prefix} - Post {current_post} of {len(post_to_process)}"
)
PixivHelper.print_and_log(None, f'Post #: {current_post}')
PixivHelper.print_and_log('info', f'Post ID : {item.imageId}')
tags_str = ', '.join(item.imageTags)
PixivHelper.print_and_log('info', f'Tags : {tags_str}')
download_post(caller, config, item)
current_post = current_post + 1
except PixivException as pex:
PixivHelper.print_and_log(
"error",
f"Failed to process PixivSketch for {artist_id}, maybe doesn't have PixivSketch? ==> {pex.message}"
)
except Exception as ex:
if isinstance(ex, KeyboardInterrupt):
raise
caller.ERROR_CODE = getattr(ex, 'errorCode', -1)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log(
'error', f'Error at process_sketch_artists(): {artist_id}')
PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')
def PrintInfo(self):
PixivHelper.safePrint('Search Result')
if self.memberId > 0:
PixivHelper.safePrint('Member Id: {0}'.format(self.memberId))
PixivHelper.safePrint('Query: {0}'.format(self.query))
PixivHelper.safePrint('haveImage : {0}'.format(self.haveImage))
PixivHelper.safePrint('urls : {0}'.format(len(self.itemList)))
for item in self.itemList:
print "\tImage Id: {0}\tFav Count:{1}".format(
item.imageId, item.bookmarkCount)
PixivHelper.safePrint('total : {0}'.format(self.availableImages))
PixivHelper.safePrint('last? : {0}'.format(self.isLastPage))
def WriteHtml(self, html_template, useAbsolutePaths, filename):
info = None
try:
PixivHelper.makeSubdirs(filename)
info = codecs.open(filename, 'wb', encoding='utf-8')
except IOError:
info = codecs.open(str(self.imageId) + ".html",
'wb',
encoding='utf-8')
PixivHelper.get_logger().exception(
"Error when saving article html: %s, file is saved to: %s.html",
filename, self.imageId)
cover_image = ""
if self.coverImageUrl:
cover_image = f'<div class="cover"><img src="{self.coverImageUrl}"/></div>'
page = html_template.replace("%coverImage%", cover_image)
page = page.replace("%coverImageUrl%", self.coverImageUrl or "")
page = page.replace("%artistName%", self.parent.artistName)
page = page.replace("%imageTitle%", self.imageTitle)
page = page.replace("%worksDate%", self.worksDate)
token_body_text = ""
token_images = ""
token_text = ""
if self.type == "article":
token_body_text = f'<div class="article">{self.body_text}</div>'
else:
token_images = '<div class="non-article images">{0}</div>'.format(
"".join([
'<a href="{0}">{1}</a>'.format(
x,
f'<img scr="{0}"/>' if x[x.rindex(".") + 1:].lower()
in ["jpg", "jpeg", "png", "bmp"] else x)
for x in self.images
]))
token_text = '<div class="non-article text">{0}</div>'.format(
"".join([
'<p>{0}</p>'.format(x.rstrip())
for x in self.body_text.split("\n")
]))
page = page.replace("%body_text(article)%", token_body_text)
page = page.replace("%images(non-article)%", token_images)
page = page.replace("%text(non-article)%", token_text)
page = BeautifulSoup(page, features="html5lib")
imageATags = page.find_all("a", attrs={"href": True})
for imageATag in imageATags:
tag = imageATag.img
if tag:
tag["src"] = imageATag["href"]
root = page.find("div", attrs={"class": "root"})
if root:
root["class"].append(
"non-article" if self.type != "article" else "article")
page = page.prettify()
html_dir = os.path.dirname(filename)
for k, v in self.linkToFile.items():
if not useAbsolutePaths:
try:
v = os.path.relpath(v, html_dir)
except ValueError:
PixivHelper.get_logger().exception(
"Error when converting local paths to relative ones, absolute paths are used",
filename, self.imageId)
v = "file://" + v
else:
v = "file://" + v
page = page.replace(k, v)
info.write(page)
info.close()
def process_manga_series(caller,
config,
manga_series_id: int,
start_page: int = 1,
end_page: int = 0,
notifier=None):
if notifier is None:
notifier = PixivHelper.dummy_notifier
try:
msg = Fore.YELLOW + Style.NORMAL + f'Processing Manga Series Id: {manga_series_id}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
notifier(type="MANGA_SERIES", message=msg)
if start_page != 1:
PixivHelper.print_and_log('info', 'Start Page: ' + str(start_page))
if end_page != 0:
PixivHelper.print_and_log('info', 'End Page: ' + str(end_page))
flag = True
current_page = start_page
while flag:
manga_series = PixivBrowserFactory.getBrowser().getMangaSeries(manga_series_id, current_page)
for (image_id, order) in manga_series.pages_with_order:
result = process_image(caller,
config,
artist=manga_series.artist,
image_id=image_id,
user_dir='',
bookmark=False,
search_tags='',
title_prefix="",
bookmark_count=-1,
image_response_count=-1,
notifier=notifier,
useblacklist=True,
manga_series_order=order,
manga_series_parent=manga_series)
PixivHelper.wait(result, config)
current_page += 1
if manga_series.is_last_page:
PixivHelper.print_and_log('info', f'Last Page {manga_series.current_page}')
flag = False
if current_page > end_page and end_page != 0:
PixivHelper.print_and_log('info', f'End Page reached {end_page}')
flag = False
if manga_series.pages_with_order is None or len(manga_series.pages_with_order) == 0:
PixivHelper.print_and_log('info', 'No more works.')
flag = False
except Exception as ex:
if isinstance(ex, KeyboardInterrupt):
raise
caller.ERROR_CODE = getattr(ex, 'errorCode', -1)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log('error', f'Error at process_manga_series(): {manga_series_id}')
PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')
raise
class PixivConfig():
'''Configuration class'''
__logger = PixivHelper.get_logger()
configFileLocation = "config.ini"
__items = [
ConfigItem("Network", "useProxy", False),
ConfigItem("Network", "proxyAddress", ""),
ConfigItem("Network", "useragent",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36"),
ConfigItem("Network", "useRobots", True),
ConfigItem("Network", "timeout", 60),
ConfigItem("Network", "retry", 3),
ConfigItem("Network", "retryWait", 5),
ConfigItem("Network", "downloadDelay", 2),
ConfigItem("Network", "checkNewVersion", True),
ConfigItem("Network", "openNewVersion", True),
ConfigItem("Network", "enableSSLVerification", True),
ConfigItem("Debug", "logLevel", "DEBUG",
followup=lambda x: x.upper(),
restriction=lambda x: x.upper() in ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG',
'NOTSET']),
ConfigItem("Debug", "enableDump", True),
ConfigItem("Debug", "skipDumpFilter", ""),
ConfigItem("Debug", "dumpMediumPage", False),
ConfigItem("Debug", "dumpTagSearchPage", False),
ConfigItem("Debug", "debugHttp", False),
ConfigItem("IrfanView", "IrfanViewPath", r"C:\Program Files\IrfanView", followup=os.path.expanduser),
ConfigItem("IrfanView", "startIrfanView", False),
ConfigItem("IrfanView", "startIrfanSlide", False),
ConfigItem("IrfanView", "createDownloadLists", False),
ConfigItem("Settings", "downloadListDirectory", ".", followup=os.path.expanduser),
ConfigItem("Settings", "useList", False),
ConfigItem("Settings", "processFromDb", True),
ConfigItem("Settings", "rootDirectory", "."),
ConfigItem("Settings", "downloadAvatar", False),
ConfigItem("Settings", "useSuppressTags", False),
ConfigItem("Settings", "tagsLimit", -1),
ConfigItem("Settings", "writeImageInfo", False),
ConfigItem("Settings", "writeImageJSON", False),
ConfigItem("Settings", "writeRawJSON", False),
ConfigItem("Settings", "RawJSONFilter", "id,title,description,alt,userIllusts,storableTags,zoneConfig,extraData,comicPromotion,fanboxPromotion"),
ConfigItem("Settings", "includeSeriesJSON", False),
ConfigItem("Settings", "verifyImage", False),
ConfigItem("Settings", "writeUrlInDescription", False),
ConfigItem("Settings", "urlBlacklistRegex", ""),
ConfigItem("Settings", "dbPath", ""),
ConfigItem("Settings", "setLastModified", True),
ConfigItem("Settings", "useLocalTimezone", False),
ConfigItem("Filename",
"filenameFormat",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("Filename",
"filenameMangaFormat",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0 and (x.find("%urlFilename%") >= 0 or (x.find('%page_index%') >= 0 or x.find('%page_number%') >= 0)),
error_message="At least %urlFilename%, %page_index%, or %page_number% is required in"),
ConfigItem("Filename", "filenameInfoFormat",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("Filename", "filenameMangaInfoFormat",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("Filename", "filenameSeriesJSON",
"%artist% (%member_id%)" + os.sep + "%manga_series_id% - %manga_series_title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("Filename", "filenameFormatSketch", "%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("Filename", "avatarNameFormat", ""),
ConfigItem("Filename", "backgroundNameFormat", ""),
ConfigItem("Filename", "tagsSeparator", ", "),
ConfigItem("Filename", "createMangaDir", False),
ConfigItem("Filename", "useTagsAsDir", False),
ConfigItem("Filename", "urlDumpFilename", "url_list_%Y%m%d"),
ConfigItem("Filename", "useTranslatedTag", False),
ConfigItem("Filename", "tagTranslationLocale", "en"),
ConfigItem("Authentication", "username", ""),
ConfigItem("Authentication", "password", ""),
ConfigItem("Authentication", "cookie", ""),
ConfigItem("Authentication", "cookieFanbox", ""),
ConfigItem("Authentication", "refresh_token", ""),
ConfigItem("Pixiv", "numberOfPage", 0),
ConfigItem("Pixiv", "r18mode", False),
ConfigItem("Pixiv", "dateFormat", ""),
ConfigItem("Pixiv", "autoAddMember", False),
ConfigItem("FANBOX", "filenameFormatFanboxCover",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("FANBOX", "filenameFormatFanboxContent",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0 and (x.find("%urlFilename%") >= 0 or (x.find('%page_index%') >= 0 or x.find('%page_number%') >= 0)),
error_message="At least %urlFilename%, %page_index%, or %page_number% is required in"),
ConfigItem("FANBOX", "filenameFormatFanboxInfo",
"%artist% (%member_id%)" + os.sep + "%urlFilename% - %title%",
restriction=lambda x: x is not None and len(x) > 0),
ConfigItem("FANBOX", "writeHtml", False),
ConfigItem("FANBOX", "minTextLengthForNonArticle", 45),
ConfigItem("FANBOX", "minImageCountForNonArticle", 3),
ConfigItem("FANBOX", "useAbsolutePathsInHtml", False),
ConfigItem("FANBOX", "downloadCoverWhenRestricted", False),
ConfigItem("FANBOX", "checkDBProcessHistory", False),
ConfigItem("FANBOX", "listPathFanbox", "listfanbox.txt"),
ConfigItem("FFmpeg", "ffmpeg", "ffmpeg"),
ConfigItem("FFmpeg", "ffmpegCodec", "libvpx-vp9"),
ConfigItem("FFmpeg", "ffmpegExt", "webm"),
ConfigItem("FFmpeg", "ffmpegParam", "-lossless 1 -vsync 2 -r 999 -pix_fmt yuv420p"),
ConfigItem("FFmpeg", "webpCodec", "libwebp"),
ConfigItem("FFmpeg", "webpParam", "-lossless 0 -q:v 90 -loop 0 -vsync 2 -r 999"),
ConfigItem("Ugoira", "writeUgoiraInfo", False),
ConfigItem("Ugoira", "createUgoira", False),
ConfigItem("Ugoira", "deleteZipFile", False),
ConfigItem("Ugoira", "createGif", False),
ConfigItem("Ugoira", "createApng", False),
ConfigItem("Ugoira", "deleteUgoira", False),
ConfigItem("Ugoira", "createWebm", False),
ConfigItem("Ugoira", "createWebp", False),
ConfigItem("DownloadControl", "minFileSize", 0),
ConfigItem("DownloadControl", "maxFileSize", 0),
ConfigItem("DownloadControl", "overwrite", False),
ConfigItem("DownloadControl", "backupOldFile", False),
ConfigItem("DownloadControl", "dayLastUpdated", 7),
ConfigItem("DownloadControl", "alwaysCheckFileSize", False),
ConfigItem("DownloadControl", "checkUpdatedLimit", 0),
ConfigItem("DownloadControl", "useBlacklistTags", False),
ConfigItem("DownloadControl", "useBlacklistTitles", False),
ConfigItem("DownloadControl", "useBlacklistTitlesRegex", False),
ConfigItem("DownloadControl", "dateDiff", 0),
ConfigItem("DownloadControl", "enableInfiniteLoop", False),
ConfigItem("DownloadControl", "useBlacklistMembers", False),
ConfigItem("DownloadControl", "downloadResized", False),
ConfigItem("DownloadControl", "checkLastModified", True),
ConfigItem("DownloadControl", "skipUnknownSize", False),
]
proxy = {"http": "", "https": "", }
def __init__(self):
for item in self.__items:
setattr(self, item.option, item.default)
self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress}
def loadConfig(self, path=None):
if path is not None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print('Reading', self.configFileLocation, '...')
config = configparser.RawConfigParser()
try:
with PixivHelper.open_text_file(self.configFileLocation) as reader:
content = reader.read()
except BaseException:
print('Error at loadConfig() reading file:', self.configFileLocation, "\n", sys.exc_info())
self.__logger.exception('Error at loadConfig() reading file: %s', self.configFileLocation)
self.writeConfig(error=True, path=self.configFileLocation)
return
haveError = False
config.read_string(content)
for item in PixivConfig.__items:
option_type = type(item.default)
method = config.get
if option_type == int:
method = config.getint
elif option_type == bool:
method = config.getboolean
value = None
try:
try:
value = method(item.section, item.option)
except (configparser.NoSectionError, configparser.NoOptionError):
haveError = True
for section in config.sections():
try:
value = method(section, item.option)
break
except (configparser.NoSectionError, configparser.NoOptionError):
continue
if value is None:
raise
except BaseException:
print(item.option, "=", item.default)
value = item.default
haveError = True
# Issue #743
try:
value = item.process_value(value)
except ValueError:
print(Fore.RED + Style.BRIGHT + f"{sys.exc_info()}" + Style.RESET_ALL)
self.__logger.exception('Error at process_value() of : %s', item.option)
print(Fore.YELLOW + Style.BRIGHT + f"{item.option} = {item.default}" + Style.RESET_ALL)
value = item.default
haveError = True
# assign the value to the actual configuration attribute
self.__setattr__(item.option, value)
self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress}
if haveError:
print(Fore.RED + Style.BRIGHT + 'Configurations with invalid value are set to default value.' + Style.RESET_ALL)
self.writeConfig(error=True, path=self.configFileLocation)
print('Configuration loaded.')
# -UI01B------write config
def writeConfig(self, error=False, path=None):
'''Backup old config if exist and write updated config.ini'''
print('Writing config file...', end=' ')
config = configparser.RawConfigParser()
groups = itertools.groupby(PixivConfig.__items, lambda x: x.section)
for k, g in groups:
config.add_section(k)
for item in g:
config.set(item.section, item.option, self.__getattribute__(item.option))
if path is not None:
configlocation = path
else:
configlocation = 'config.ini'
try:
# with codecs.open('config.ini.bak', encoding = 'utf-8', mode = 'wb') as configfile:
with open(configlocation + '.tmp', 'w', encoding='utf8') as configfile:
config.write(configfile)
configfile.close()
if os.path.exists(configlocation):
if error:
backupName = configlocation + '.error-' + str(int(time.time()))
print("Backing up old config (error exist!) to " + backupName)
shutil.move(configlocation, backupName)
else:
print("Backing up old config to config.ini.bak")
shutil.move(configlocation, configlocation + '.bak')
self.__logger.debug(f"renaming {configlocation}.tmp to {configlocation}")
os.rename(configlocation + '.tmp', configlocation)
except BaseException:
self.__logger.exception('Error at writeConfig()')
raise
print('Configuration saved.')
def printConfig(self):
print('Configuration: ')
groups = {k: list(g) for k, g in itertools.groupby(PixivConfig.__items, lambda x: x.section)}
sections = ["Authentication", "Network", "Debug", "IrfanView", "Settings", "Filename", "Pixiv", "FANBOX",
"FFmpeg", "Ugoira", "DownloadControl"]
sections.extend([k for k in groups if k not in sections])
for section in sections:
g = groups.get(section)
if g:
print(f" [{section}]")
for item in g:
print(f" - {item.option:{25}} = {self.__getattribute__(item.option)}")
print('')
def process_image(caller,
config,
artist=None,
image_id=None,
user_dir='',
bookmark=False,
search_tags='',
title_prefix="",
bookmark_count=-1,
image_response_count=-1,
notifier=None,
useblacklist=True,
manga_series_order=-1,
manga_series_parent=None) -> int:
# caller function/method
# TODO: ideally to be removed or passed as argument
db = caller.__dbManager__
if notifier is None:
notifier = PixivHelper.dummy_notifier
# override the config source if job_option is give for filename formats
extension_filter = None
if hasattr(config, "extensionFilter"):
extension_filter = config.extensionFilter
parse_medium_page = None
image = None
result = None
referer = f'https://www.pixiv.net/artworks/{image_id}'
filename = f'no-filename-{image_id}.tmp'
try:
msg = Fore.YELLOW + Style.NORMAL + f'Processing Image Id: {image_id}' + Style.RESET_ALL
PixivHelper.print_and_log(None, msg)
notifier(type="IMAGE", message=msg)
# check if already downloaded. images won't be downloaded twice - needed in process_image to catch any download
r = db.selectImageByImageId(image_id, cols='save_name')
exists = False
in_db = False
if r is not None:
exists = db.cleanupFileExists(r[0])
in_db = True
# skip if already recorded in db and alwaysCheckFileSize is disabled and overwrite is disabled.
if in_db and not config.alwaysCheckFileSize and not config.overwrite:
PixivHelper.print_and_log(None, f'Already downloaded in DB: {image_id}')
gc.collect()
return PixivConstant.PIXIVUTIL_SKIP_DUPLICATE_NO_WAIT
# get the medium page
try:
(image, parse_medium_page) = PixivBrowserFactory.getBrowser().getImagePage(image_id=image_id,
parent=artist,
from_bookmark=bookmark,
bookmark_count=bookmark_count,
manga_series_order=manga_series_order,
manga_series_parent=manga_series_parent)
if len(title_prefix) > 0:
caller.set_console_title(f"{title_prefix} ImageId: {image.imageId}")
else:
caller.set_console_title(f"MemberId: {image.artist.artistId} ImageId: {image.imageId}")
except PixivException as ex:
caller.ERROR_CODE = ex.errorCode
caller.__errorList.append(dict(type="Image", id=str(image_id), message=ex.message, exception=ex))
if ex.errorCode == PixivException.UNKNOWN_IMAGE_ERROR:
PixivHelper.print_and_log('error', ex.message)
elif ex.errorCode == PixivException.SERVER_ERROR:
PixivHelper.print_and_log('error', f'Giving up image_id (medium): {image_id}')
elif ex.errorCode > 2000:
PixivHelper.print_and_log('error', f'Image Error for {image_id}: {ex.message}')
if parse_medium_page is not None:
dump_filename = f'Error medium page for image {image_id}.html'
PixivHelper.dump_html(dump_filename, parse_medium_page)
PixivHelper.print_and_log('error', f'Dumping html to: {dump_filename}')
else:
PixivHelper.print_and_log('error', f'Image ID ({image_id}): {ex}')
PixivHelper.print_and_log('error', f'Stack Trace: {sys.exc_info()}')
return PixivConstant.PIXIVUTIL_NOT_OK
except Exception as ex:
PixivHelper.print_and_log('error', f'Image ID ({image_id}): {ex}')
if parse_medium_page is not None:
dump_filename = f'Error medium page for image {image_id}.html'
PixivHelper.dump_html(dump_filename, parse_medium_page)
PixivHelper.print_and_log('error', f'Dumping html to: {dump_filename}')
PixivHelper.print_and_log('error', f'Stack Trace: {sys.exc_info()}')
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
return PixivConstant.PIXIVUTIL_NOT_OK
download_image_flag = True
# date validation and blacklist tag validation
if config.dateDiff > 0:
if image.worksDateDateTime != datetime.datetime.fromordinal(1).replace(tzinfo=datetime_z.utc):
if image.worksDateDateTime < (datetime.datetime.today() - datetime.timedelta(config.dateDiff)).replace(tzinfo=datetime_z.utc):
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} – it\'s older than: {config.dateDiff} day(s).')
download_image_flag = False
result = PixivConstant.PIXIVUTIL_SKIP_OLDER
if useblacklist:
if config.useBlacklistMembers and download_image_flag:
if str(image.originalArtist.artistId) in caller.__blacklistMembers:
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} – blacklisted member id: {image.originalArtist.artistId}')
download_image_flag = False
result = PixivConstant.PIXIVUTIL_SKIP_BLACKLIST
if config.useBlacklistTags and download_image_flag:
for item in caller.__blacklistTags:
if item in image.imageTags:
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} – blacklisted tag: {item}')
download_image_flag = False
result = PixivConstant.PIXIVUTIL_SKIP_BLACKLIST
break
if config.useBlacklistTitles and download_image_flag:
if config.useBlacklistTitlesRegex:
for item in caller.__blacklistTitles:
if re.search(rf"{item}", image.imageTitle):
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} – Title matched: {item}')
download_image_flag = False
result = PixivConstant.PIXIVUTIL_SKIP_BLACKLIST
break
else:
for item in caller.__blacklistTitles:
if item in image.imageTitle:
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} – Title contained: {item}')
download_image_flag = False
result = PixivConstant.PIXIVUTIL_SKIP_BLACKLIST
break
# Issue #726
if extension_filter is not None and len(extension_filter) > 0:
for url in image.imageUrls:
ext = PixivHelper.get_extension_from_url(url)
# add alias for ugoira
if "ugoira" in extension_filter:
extension_filter = f"{extension_filter}|zip"
if re.search(extension_filter, ext) is None:
download_image_flag = False
PixivHelper.print_and_log('warn', f'Skipping image_id: {image_id} - url is not in the filter: {extension_filter} => {url}')
break
if download_image_flag:
if artist is None:
PixivHelper.print_and_log(None, f'Member Name : {image.artist.artistName}')
PixivHelper.print_and_log(None, f'Member Avatar: {image.artist.artistAvatar}')
PixivHelper.print_and_log(None, f'Member Token : {image.artist.artistToken}')
PixivHelper.print_and_log(None, f'Member Background : {image.artist.artistBackground}')
PixivHelper.print_and_log(None, f"Title: {image.imageTitle}")
tags_str = ', '.join(image.imageTags)
PixivHelper.print_and_log(None, f"Tags : {tags_str}")
PixivHelper.print_and_log(None, f"Date : {image.worksDateDateTime}")
PixivHelper.print_and_log(None, f"Mode : {image.imageMode}")
# # get bookmark count
# if ("%bookmark_count%" in config.filenameFormat or "%image_response_count%" in config.filenameFormat) and image.bookmark_count == -1:
# PixivHelper.print_and_log(None, "Parsing bookmark page", end=' ')
# bookmark_url = f'https://www.pixiv.net/bookmark_detail.php?illust_id={image_id}'
# parse_bookmark_page = PixivBrowserFactory.getBrowser().getPixivPage(bookmark_url)
# image.ParseBookmarkDetails(parse_bookmark_page)
# parse_bookmark_page.decompose()
# del parse_bookmark_page
# PixivHelper.print_and_log(None, f"Bookmark Count : {image.bookmark_count}")
# caller.__br__.back()
PixivHelper.print_and_log(None, f"Bookmark Count : {image.bookmark_count}")
if config.useSuppressTags:
for item in caller.__suppressTags:
if item in image.imageTags:
image.imageTags.remove(item)
# get manga page
if image.imageMode == 'manga':
PixivHelper.print_and_log(None, f"Page Count : {image.imageCount}")
if user_dir == '': # Yavos: use config-options
target_dir = config.rootDirectory
else: # Yavos: use filename from list
target_dir = user_dir
result = PixivConstant.PIXIVUTIL_OK
manga_files = list()
page = 0
# Issue #639
source_urls = image.imageUrls
if config.downloadResized:
source_urls = image.imageResizedUrls
# debugging purpose, to avoid actual download
if caller.DEBUG_SKIP_DOWNLOAD_IMAGE:
return PixivConstant.PIXIVUTIL_OK
for img in source_urls:
PixivHelper.print_and_log(None, f'Image URL : {img}')
url = os.path.basename(img)
split_url = url.split('.')
if split_url[0].startswith(str(image_id)):
filename_format = config.filenameFormat
if image.imageMode == 'manga':
filename_format = config.filenameMangaFormat
filename = PixivHelper.make_filename(filename_format,
image,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl=url,
bookmark=bookmark,
searchTags=search_tags,
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
filename = PixivHelper.sanitize_filename(filename, target_dir)
if image.imageMode == 'manga' and config.createMangaDir:
manga_page = __re_manga_page.findall(filename)
if len(manga_page) > 0:
splitted_filename = filename.split(manga_page[0][0], 1)
splitted_manga_page = manga_page[0][0].split("_p", 1)
# filename = splitted_filename[0] + splitted_manga_page[0] + os.sep + "_p" + splitted_manga_page[1] + splitted_filename[1]
filename = f"{splitted_filename[0]}{splitted_manga_page[0]}{os.sep}_p{splitted_manga_page[1]}{splitted_filename[1]}"
PixivHelper.print_and_log('info', f'Filename : {filename}')
result = PixivConstant.PIXIVUTIL_NOT_OK
try:
(result, filename) = PixivDownloadHandler.download_image(caller,
img,
filename,
referer,
config.overwrite,
config.retry,
config.backupOldFile,
image,
page,
notifier)
if result == PixivConstant.PIXIVUTIL_NOT_OK:
PixivHelper.print_and_log('error', f'Image url not found/failed to download: {image.imageId}')
elif result == PixivConstant.PIXIVUTIL_ABORTED:
raise KeyboardInterrupt()
manga_files.append((image_id, page, filename))
page = page + 1
except urllib.error.URLError:
PixivHelper.print_and_log('error', f'Error when download_image(), giving up url: {img}')
PixivHelper.print_and_log(None, '')
if config.writeImageXMPPerImage:
filename_info_format = config.filenameInfoFormat or config.filenameFormat
# Issue #575
if image.imageMode == 'manga':
filename_info_format = config.filenameMangaInfoFormat or config.filenameMangaFormat or filename_info_format
info_filename = PixivHelper.make_filename(filename_info_format,
image,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl=url,
appendExtension=False,
bookmark=bookmark,
searchTags=search_tags,
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
info_filename = PixivHelper.sanitize_filename(info_filename, target_dir)
image.WriteXMP(info_filename + ".xmp")
if config.writeImageInfo or config.writeImageJSON or config.writeImageXMP:
filename_info_format = config.filenameInfoFormat or config.filenameFormat
# Issue #575
if image.imageMode == 'manga':
filename_info_format = config.filenameMangaInfoFormat or config.filenameMangaFormat or filename_info_format
info_filename = PixivHelper.make_filename(filename_info_format,
image,
tagsSeparator=config.tagsSeparator,
tagsLimit=config.tagsLimit,
fileUrl=url,
appendExtension=False,
bookmark=bookmark,
searchTags=search_tags,
useTranslatedTag=config.useTranslatedTag,
tagTranslationLocale=config.tagTranslationLocale)
info_filename = PixivHelper.sanitize_filename(info_filename, target_dir)
# trim _pXXX
info_filename = re.sub(r'_p?\d+$', '', info_filename)
if config.writeImageInfo:
image.WriteInfo(info_filename + ".txt")
if config.writeImageJSON:
image.WriteJSON(info_filename + ".json", config.RawJSONFilter)
if config.includeSeriesJSON and image.seriesNavData and image.seriesNavData['seriesId'] not in caller.__seriesDownloaded:
json_filename = PixivHelper.make_filename(config.filenameSeriesJSON,
image,
fileUrl=url,
appendExtension=False
)
json_filename = PixivHelper.sanitize_filename(json_filename, target_dir)
# trim _pXXX
json_filename = re.sub(r'_p?\d+$', '', json_filename)
image.WriteSeriesData(image.seriesNavData['seriesId'], caller.__seriesDownloaded, json_filename + ".json")
if config.writeImageXMP and not config.writeImageXMPPerImage:
image.WriteXMP(info_filename + ".xmp")
if image.imageMode == 'ugoira_view':
if config.writeUgoiraInfo:
image.WriteUgoiraData(filename + ".js")
# Handle #451
if config.createUgoira and (result in (PixivConstant.PIXIVUTIL_OK, PixivConstant.PIXIVUTIL_SKIP_DUPLICATE)):
PixivDownloadHandler.handle_ugoira(image, filename, config, notifier)
if config.writeUrlInDescription:
PixivHelper.write_url_in_description(image, config.urlBlacklistRegex, config.urlDumpFilename)
if in_db and not exists:
result = PixivConstant.PIXIVUTIL_CHECK_DOWNLOAD # There was something in the database which had not been downloaded
# Only save to db if all images is downloaded completely
if result in (PixivConstant.PIXIVUTIL_OK,
PixivConstant.PIXIVUTIL_SKIP_DUPLICATE,
PixivConstant.PIXIVUTIL_SKIP_LOCAL_LARGER):
try:
db.insertImage(image.artist.artistId, image.imageId, image.imageMode)
except BaseException:
PixivHelper.print_and_log('error', f'Failed to insert image id:{image.imageId} to DB')
db.updateImage(image.imageId, image.imageTitle, filename, image.imageMode)
if len(manga_files) > 0:
db.insertMangaImages(manga_files)
# map back to PIXIVUTIL_OK (because of ugoira file check)
result = 0
if image is not None:
del image
if parse_medium_page is not None:
del parse_medium_page
gc.collect()
PixivHelper.print_and_log(None, '\n')
return result
except Exception as ex:
if isinstance(ex, KeyboardInterrupt):
raise
caller.ERROR_CODE = getattr(ex, 'errorCode', -1)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log('error', f'Error at process_image(): {image_id}')
PixivHelper.print_and_log('error', f'Exception: {sys.exc_info()}')
if parse_medium_page is not None:
dump_filename = f'Error medium page for image {image_id}.html'
PixivHelper.dump_html(dump_filename, parse_medium_page)
PixivHelper.print_and_log('error', f'Dumping html to: {dump_filename}')
raise
def main(self):
try:
while True:
selection = self.menu()
if selection == '1':
self.printMemberList()
elif selection == '2':
self.printImageList()
elif selection == '3':
filename = raw_input('Filename? ')
includeArtistToken = raw_input(
'Include Artist Token[y/n]? ')
if includeArtistToken.lower() == 'y':
includeArtistToken = True
else:
includeArtistToken = False
self.exportList(filename, includeArtistToken)
elif selection == '4':
filename = raw_input('Filename? ')
self.exportDetailedList(filename)
elif selection == '5':
date = raw_input('Number of date? ')
rows = self.selectMembersByLastDownloadDate(date)
if rows != None:
for row in rows:
for string in row:
print ' ',
PixivHelper.safePrint(unicode(string), False)
print '\n'
else:
print 'Not Found!\n'
elif selection == '6':
image_id = raw_input('image_id? ')
row = self.selectImageByImageId(image_id)
if row != None:
for string in row:
print ' ',
PixivHelper.safePrint(unicode(string), False)
print '\n'
else:
print 'Not Found!\n'
elif selection == '7':
member_id = raw_input('member_id? ')
row = self.selectMemberByMemberId(member_id)
if row != None:
for string in row:
print ' ',
PixivHelper.safePrint(unicode(string), False)
print '\n'
else:
print 'Not Found!\n'
elif selection == '8':
member_id = raw_input('member_id? ')
rows = self.selectImageByMemberId(member_id)
if rows != None:
for row in rows:
for string in row:
print ' ',
PixivHelper.safePrint(unicode(string), False)
print '\n'
else:
print 'Not Found!\n'
elif selection == '9':
member_id = raw_input('member_id? ')
self.deleteMemberByMemberId(member_id)
elif selection == '10':
image_id = raw_input('image_id? ')
self.deleteImage(image_id)
elif selection == '11':
member_id = raw_input('member_id? ')
self.deleteCascadeMemberByMemberId(member_id)
elif selection == '12':
member_id = raw_input('member_id? ')
image_id = raw_input('image_id? ')
self.blacklistImage(member_id, image_id)
elif selection == '13':
self.printMemberList(isDeleted=True)
elif selection == 'c':
self.cleanUp()
elif selection == 'p':
self.compactDatabase()
elif selection == 'r':
self.replaceRootPath()
elif selection == 'x':
break
print 'end PixivDBManager.'
except:
print 'Error: ', sys.exc_info()
##raw_input('Press enter to exit.')
self.main()
def main(self):
PixivHelper.get_logger().info('DB Manager (d).')
try:
while True:
selection = self.menu()
if selection == '1':
self.printMemberList()
elif selection == '2':
self.printImageList()
elif selection == '3':
filename = input('Filename? ').rstrip("\r")
includeArtistToken = input(
'Include Artist Token[y/n]? ').rstrip("\r")
if includeArtistToken.lower() == 'y':
includeArtistToken = True
else:
includeArtistToken = False
self.exportList(filename, includeArtistToken)
elif selection == '4':
filename = input('Filename? ').rstrip("\r")
self.exportDetailedList(filename)
elif selection == '5':
date = input('Number of date? ').rstrip("\r")
rows = self.selectMembersByLastDownloadDate(date)
if rows is not None:
for row in rows:
print("{0}\t\t{1}\n".format(
row.memberId, row.path))
else:
print('Not Found!\n')
elif selection == '6':
image_id = input('image_id? ').rstrip("\r")
row = self.selectImageByImageId(image_id)
if row is not None:
for string in row:
print(' ', end=' ')
print(string)
print('\n')
else:
print('Not Found!\n')
elif selection == '7':
member_id = input('member_id? ').rstrip("\r")
row = self.selectMemberByMemberId(member_id)
if row is not None:
for string in row:
print(' ', end=' ')
print(string)
print('\n')
else:
print('Not Found!\n')
elif selection == '8':
member_id = input('member_id? ').rstrip("\r")
rows = self.selectImageByMemberId(member_id)
if rows is not None:
for row in rows:
for string in row:
print(' ', end=' ')
print(string)
print('\n')
else:
print('Not Found!\n')
elif selection == '9':
member_id = input('member_id? ').rstrip("\r")
self.deleteMemberByMemberId(member_id)
elif selection == '10':
image_id = input('image_id? ').rstrip("\r")
self.deleteImage(image_id)
elif selection == '11':
member_id = input('member_id? ').rstrip("\r")
self.deleteCascadeMemberByMemberId(member_id)
elif selection == '12':
member_id = input('member_id? ').rstrip("\r")
image_id = input('image_id? ').rstrip("\r")
self.blacklistImage(member_id, image_id)
elif selection == '13':
self.printMemberList(isDeleted=True)
elif selection == 'f1':
filename = input('Filename? ').rstrip("\r")
sep = input(
'Separator? (1(default)=",", 2="\\t") ').rstrip("\r")
sep = "\t" if sep == "2" else ","
self.exportFanboxPostList(filename, sep)
elif selection == 'f2':
member_id = input('member_id? ').rstrip("\r")
self.deleteFanboxPost(member_id, "member_id")
elif selection == 'f3':
post_id = input('post_id? ').rstrip("\r")
self.deleteFanboxPost(post_id, "post_id")
elif selection == 'c':
self.cleanUp()
self.cleanUpFanbox()
elif selection == 'i':
self.interactiveCleanUp()
self.interactiveCleanUpFanbox()
elif selection == 'p':
self.compactDatabase()
elif selection == 'r':
self.replaceRootPath()
elif selection == 'x':
break
print('end PixivDBManager.')
except BaseException:
print('Error: ', sys.exc_info())
self.main()
def loadConfig(self, path=None):
''' new settings must be added on the last'''
if path is not None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print('Reading', self.configFileLocation, '...')
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication', 'username')
self.password = config.get('Authentication', 'password')
self.cookie = config.get('Authentication', 'cookie')
self.tagsSeparator = PixivHelper.toUnicode(
config.get('Settings', 'tagsseparator'),
encoding=sys.stdin.encoding)
self.rootDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings', 'rootdirectory'),
encoding=sys.stdin.encoding))
try:
self.IrfanViewPath = os.path.expanduser(
PixivHelper.toUnicode(config.get('IrfanView',
'IrfanViewPath'),
encoding=sys.stdin.encoding))
self.downloadListDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings',
'downloadListDirectory'),
encoding=sys.stdin.encoding))
except BaseException:
pass
try:
self.processFromDb = config.getboolean('Settings',
'processfromdb')
except ValueError:
print("processFromDb = True")
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings',
'daylastupdated')
except ValueError:
print("dayLastUpdated = 7")
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings', 'datediff')
except ValueError:
print("dateDiff = 0")
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Network', 'proxyaddress')
except ValueError:
print("proxyAddress = ''")
self.proxyAddress = ''
haveError = True
self.proxy = {
'http': self.proxyAddress,
'https': self.proxyAddress
}
try:
self.useProxy = config.getboolean('Network', 'useproxy')
except ValueError:
print("useProxy = False")
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings', 'uselist')
except ValueError:
print("useList = False")
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv', 'r18mode')
except ValueError:
print("r18mode = False")
self.r18mode = False
haveError = True
_useragent = config.get('Network', 'useragent')
if _useragent is not None:
self.useragent = _useragent
_filenameFormat = config.get('Settings', 'filenameformat')
_filenameFormat = PixivHelper.toUnicode(
_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat is not None and len(_filenameFormat) > 0:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Settings',
'filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(
_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat is not None and len(
_filenameMangaFormat) > 0:
# check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find(
'%page_index%'
) == -1 and _filenameMangaFormat.find(
'%page_number%') == -1:
print(
'No page identifier, appending %page_index% to the filename manga format.'
)
_filenameMangaFormat = _filenameMangaFormat + unicode(
' %page_index%')
print("_filenameMangaFormat =", _filenameMangaFormat)
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
_filenameInfoFormat = config.get('Settings', 'filenameinfoformat')
_filenameInfoFormat = PixivHelper.toUnicode(
_filenameInfoFormat, encoding=sys.stdin.encoding)
if _filenameInfoFormat is not None and len(
_filenameInfoFormat) > 0:
self.filenameInfoFormat = _filenameInfoFormat
try:
self.debugHttp = config.getboolean('Debug', 'debughttp')
except ValueError:
self.debugHttp = False
print("debugHttp = False")
haveError = True
try:
self.useRobots = config.getboolean('Network', 'userobots')
except ValueError:
self.useRobots = False
print("useRobots = False")
haveError = True
try:
self.overwrite = config.getboolean('Settings', 'overwrite')
except ValueError:
print("overwrite = False")
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean(
'Settings', 'createMangaDir')
except ValueError:
print("createMangaDir = False")
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Network', 'timeout')
except ValueError:
print("timeout = 60")
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Network', 'retry')
except ValueError:
print("retry = 3")
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Network', 'retrywait')
except ValueError:
print("retryWait = 5")
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv', 'numberofpage')
except ValueError:
self.numberOfPage = 0
print("numberOfPage = 0")
haveError = True
try:
self.createDownloadLists = config.getboolean(
'IrfanView', 'createDownloadLists')
except ValueError:
self.createDownloadLists = False
print("createDownloadLists = False")
haveError = True
try:
self.startIrfanView = config.getboolean(
'IrfanView', 'startIrfanView')
except ValueError:
self.startIrfanView = False
print("startIrfanView = False")
haveError = True
try:
self.startIrfanSlide = config.getboolean(
'IrfanView', 'startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print("startIrfanSlide = False")
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean(
'Settings', 'alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print("alwaysCheckFileSize = False")
haveError = True
try:
self.downloadAvatar = config.getboolean(
'Settings', 'downloadAvatar')
except ValueError:
self.downloadAvatar = False
print("downloadAvatar = False")
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings',
'checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print("checkUpdatedLimit = 0")
haveError = True
try:
self.useTagsAsDir = config.getboolean('Settings',
'useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print("useTagsAsDir = False")
haveError = True
try:
self.useBlacklistTags = config.getboolean(
'Settings', 'useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print("useBlacklistTags = False")
haveError = True
try:
self.useSuppressTags = config.getboolean(
'Settings', 'useSuppressTags')
except ValueError:
self.useSuppressTags = False
print("useSuppressTags = False")
haveError = True
try:
self.tagsLimit = config.getint('Settings', 'tagsLimit')
except ValueError:
self.tagsLimit = -1
print("tagsLimit = -1")
haveError = True
try:
self.writeImageInfo = config.getboolean(
'Settings', 'writeImageInfo')
except ValueError:
self.writeImageInfo = False
print("writeImageInfo = False")
haveError = True
try:
self.backupOldFile = config.getboolean('Settings',
'backupOldFile')
except ValueError:
self.backupOldFile = False
print("backupOldFile = False")
haveError = True
try:
self.logLevel = config.get('Debug', 'logLevel').upper()
if self.logLevel not in [
'CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO',
'DEBUG', 'NOTSET'
]:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print("logLevel = DEBUG")
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Debug', 'enableDump')
except ValueError:
print("enableDump = True")
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Debug', 'skipDumpFilter')
except ValueError:
print("skipDumpFilter = ''")
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean(
'Debug', 'dumpMediumPage')
except ValueError:
print("dumpMediumPage = False")
self.dumpMediumPage = False
haveError = True
try:
self.enableInfiniteLoop = config.getboolean(
'Settings', 'enableInfiniteLoop')
except ValueError:
self.enableInfiniteLoop = False
print("enableInfiniteLoop = False")
haveError = True
try:
self.dumpTagSearchPage = config.getboolean(
'Debug', 'dumpTagSearchPage')
except ValueError:
self.dumpTagSearchPage = False
print("dumpTagSearchPage = False")
haveError = True
try:
self.dateFormat = config.get('Pixiv', 'dateFormat')
except ValueError:
print("dateFormat = ''")
self.dateFormat = ''
haveError = True
try:
self.verifyImage = config.getboolean('Settings', 'verifyImage')
except ValueError:
print("verifyImage = False")
self.verifyImage = False
haveError = True
try:
self.writeUrlInDescription = config.getboolean(
'Settings', 'writeUrlInDescription')
except ValueError:
print("writeUrlInDescription = False")
self.writeUrlInDescription = False
haveError = True
try:
self.urlBlacklistRegex = config.get('Settings',
'urlBlacklistRegex')
except ValueError:
print("urlBlacklistRegex = ")
self.urlBlacklistRegex = ""
haveError = True
try:
self.urlDumpFilename = config.get('Settings',
'urlDumpFilename')
except ValueError:
print("urlDumpFilename = url_list_%Y%m%d")
self.urlDumpFilename = "url_list_%Y%m%d"
haveError = True
try:
self.dbPath = config.get('Settings', 'dbPath')
except ValueError:
print("dbPath = ''")
self.dbPath = ''
haveError = True
try:
self.useBlacklistMembers = config.getboolean(
'Settings', 'useBlacklistMembers')
except ValueError:
print("useBlacklistMembers = False")
self.useBlacklistMembers = False
haveError = True
try:
self.avatarNameFormat = config.get('Settings',
'avatarNameFormat')
self.avatarNameFormat = PixivHelper.toUnicode(
self.avatarNameFormat, encoding=sys.stdin.encoding)
except ValueError:
print("avatarNameFormat = ")
self.avatarNameFormat = ""
haveError = True
try:
self.writeImageJSON = config.getboolean(
'Settings', 'writeImageJSON')
except ValueError:
self.writeImageJSON = False
print("writeImageJSON = False")
haveError = True
try:
self.downloadDelay = config.getint('Network', 'downloadDelay')
except ValueError:
self.downloadDelay = 2
print("downloadDelay = 2")
haveError = True
try:
self.ffmpeg = config.get('FFmpeg', 'ffmpeg')
except ValueError:
print("ffmpeg = 'ffmpeg'")
self.ffmpeg = 'ffmpeg'
haveError = True
try:
self.ffmpegCodec = config.get('FFmpeg', 'ffmpegCodec')
except ValueError:
print("ffmpegCodec = 'libvpx-vp9'")
self.ffmpegCodec = 'libvpx-vp9'
haveError = True
try:
self.ffmpegParam = config.get('FFmpeg', 'ffmpegParam')
except ValueError:
print("ffmpegParam = '-lossless 1'")
self.ffmpegParam = '-lossless 1'
haveError = True
try:
self.setLastModified = config.getboolean(
'Settings', 'setLastModified')
except ValueError:
print("setLastModified = True")
self.setLastModified = True
haveError = True
try:
self.useLocalTimezone = config.getboolean(
'Settings', 'useLocalTimezone')
except ValueError:
print("useLocalTimezone = False")
self.useLocalTimezone = False
haveError = True
try:
self.checkNewVersion = config.getboolean(
'Network', 'checkNewVersion')
except ValueError:
print("checkNewVersion = True")
self.checkNewVersion = True
haveError = True
try:
self.webpCodec = config.get('FFmpeg', 'webpCodec')
except ValueError:
print("webpCodec = 'libwebp'")
self.webpCodec = 'libwebp'
haveError = True
try:
self.webpParam = config.get('FFmpeg', 'webpParam')
except ValueError:
print("webpParam = '-lossless 0 -q:v 90'")
self.webpParam = '-lossless 0 -q:v 90'
haveError = True
try:
self.createWebp = config.getboolean('Ugoira', 'createWebp')
except ValueError:
print("createWebp = False")
self.createWebp = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean(
'Ugoira', 'writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print("writeUgoiraInfo = False")
haveError = True
try:
self.createUgoira = config.getboolean('Ugoira', 'createUgoira')
except ValueError:
self.createUgoira = False
print("createUgoira = False")
haveError = True
try:
self.deleteZipFile = config.getboolean('Ugoira',
'deleteZipFile')
except ValueError:
self.deleteZipFile = False
print("deleteZipFile = False")
haveError = True
try:
self.createGif = config.getboolean('Ugoira', 'createGif')
except ValueError:
print("createGif = False")
self.createGif = False
haveError = True
try:
self.createApng = config.getboolean('Ugoira', 'createApng')
except ValueError:
print("createApng = False")
self.createApng = False
haveError = True
try:
self.deleteUgoira = config.getboolean('Ugoira', 'deleteUgoira')
except ValueError:
print("deleteUgoira = False")
self.deleteUgoira = False
haveError = True
try:
self.createWebm = config.getboolean('Ugoira', 'createWebm')
except ValueError:
print("createWebm = False")
self.createWebm = False
haveError = True
except BaseException:
print('Error at loadConfig():', sys.exc_info())
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print(
'Some configuration have invalid value, replacing with the default value.'
)
self.writeConfig(error=True)
print('done.')
#!C:/Python37-32/python
# -*- coding: utf-8 -*-
import codecs
import os
import re
import sqlite3
import sys
from datetime import datetime
import PixivHelper
from PixivListItem import PixivListItem
script_path = PixivHelper.module_path()
class PixivDBManager(object):
"""Pixiv Database Manager"""
rootDirectory = "."
def __init__(self, root_directory, target='', timeout=5 * 60):
if target is None or len(target) == 0:
target = script_path + os.sep + "db.sqlite"
PixivHelper.print_and_log('info',
"Using default DB Path: " + target)
else:
PixivHelper.print_and_log('info',
"Using custom DB Path: " + target)
self.rootDirectory = root_directory
self.conn = sqlite3.connect(target, timeout)
class PixivConfig:
'''Configuration class'''
__logger = PixivHelper.GetLogger()
configFileLocation = "config.ini"
## default value
proxyAddress = ''
proxy = {
'http': proxyAddress,
'https': proxyAddress,
}
useProxy = False
username = ''
password = ''
useragent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36'
debugHttp = False
numberOfPage = 0
useRobots = True
filenameFormat = unicode('%artist% (%member_id%)' + os.sep +
'%urlFilename% - %title%')
filenameMangaFormat = unicode('%artist% (%member_id%)' + os.sep +
'%urlFilename% - %title%')
rootDirectory = unicode('.')
overwrite = False
timeout = 60
useList = False
processFromDb = True
dayLastUpdated = 7
tagsSeparator = unicode(', ')
retry = 3
retryWait = 5
alwaysCheckFileSize = False
checkUpdatedLimit = 0
downloadAvatar = True
cookie = ''
createMangaDir = False
useTagsAsDir = False
useBlacklistTags = False
useSuppressTags = False
tagsLimit = -1
useSSL = False
writeImageInfo = False
r18mode = False
dateDiff = 0
keepSignedIn = 0
#Yavos: added next three lines
createDownloadLists = False
downloadListDirectory = unicode('.')
startIrfanView = False
startIrfanSlide = False
IrfanViewPath = unicode('C:\Program Files\IrfanView')
backupOldFile = False
logLevel = "DEBUG"
enableDump = True
skipDumpFilter = ""
dumpMediumPage = False
writeUgoiraInfo = False
createUgoira = False
deleteZipFile = False
def loadConfig(self, path=None):
if path != None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print 'Reading', self.configFileLocation, '...'
oldSetting = False
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication', 'username')
self.password = config.get('Authentication', 'password')
self.cookie = config.get('Authentication', 'cookie')
self.tagsSeparator = PixivHelper.toUnicode(
config.get('Settings', 'tagsseparator'),
encoding=sys.stdin.encoding)
self.rootDirectory = PixivHelper.toUnicode(
config.get('Settings', 'rootdirectory'),
encoding=sys.stdin.encoding)
try:
self.IrfanViewPath = PixivHelper.toUnicode(
config.get('Settings', 'IrfanViewPath'),
encoding=sys.stdin.encoding)
self.downloadListDirectory = PixivHelper.toUnicode(
config.get('Settings', 'downloadListDirectory'),
encoding=sys.stdin.encoding)
except:
pass
try:
self.processFromDb = config.getboolean('Settings',
'processfromdb')
except ValueError:
print "processFromDb = True"
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings',
'daylastupdated')
except ValueError:
print "dayLastUpdated = 7"
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings', 'datediff')
except ValueError:
print "dateDiff = 0"
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Settings', 'proxyaddress')
except ValueError:
print "proxyAddress = ''"
self.proxyAddress = ''
haveError = True
self.proxy = {
'http': self.proxyAddress,
'https': self.proxyAddress
}
try:
self.useProxy = config.getboolean('Settings', 'useproxy')
except ValueError:
print "useProxy = False"
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings', 'uselist')
except ValueError:
print "useList = False"
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv', 'r18mode')
except ValueError:
print "r18mode = False"
self.r18mode = False
haveError = True
_useragent = config.get('Settings', 'useragent')
if _useragent != None:
self.useragent = _useragent
_filenameFormat = config.get('Settings', 'filenameformat')
_filenameFormat = PixivHelper.toUnicode(
_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat != None:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Settings',
'filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(
_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat != None:
## check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find(
'%page_index%'
) == -1 and _filenameMangaFormat.find(
'%page_number%') == -1:
print 'No page identifier, appending %page_index% to the filename manga format.'
_filenameMangaFormat = _filenameMangaFormat + unicode(
' %page_index%')
print "_filenameMangaFormat =", _filenameMangaFormat
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
try:
self.debugHttp = config.getboolean('Settings', 'debughttp')
except ValueError:
self.debugHttp = False
print "debugHttp = False"
haveError = True
try:
self.useRobots = config.getboolean('Settings', 'userobots')
except ValueError:
self.useRobots = False
print "useRobots = False"
haveError = True
try:
self.overwrite = config.getboolean('Settings', 'overwrite')
except ValueError:
print "overwrite = False"
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean(
'Settings', 'createMangaDir')
except ValueError:
print "createMangaDir = False"
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Settings', 'timeout')
except ValueError:
print "timeout = 60"
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Settings', 'retry')
except ValueError:
print "retry = 3"
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Settings', 'retrywait')
except ValueError:
print "retryWait = 5"
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv', 'numberofpage')
except ValueError:
self.numberOfPage = 0
print "numberOfPage = 0"
haveError = True
try:
self.createDownloadLists = config.getboolean(
'Settings', 'createDownloadLists')
except ValueError:
self.createDownloadLists = False
print "createDownloadLists = False"
haveError = True
try:
self.startIrfanView = config.getboolean(
'Settings', 'startIrfanView')
except ValueError:
self.startIrfanView = False
print "startIrfanView = False"
haveError = True
try:
self.startIrfanSlide = config.getboolean(
'Settings', 'startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print "startIrfanSlide = False"
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean(
'Settings', 'alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print "alwaysCheckFileSize = False"
haveError = True
try:
self.downloadAvatar = config.getboolean(
'Settings', 'downloadAvatar')
except ValueError:
self.downloadAvatar = False
print "downloadAvatar = False"
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings',
'checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print "checkUpdatedLimit = 0"
haveError = True
try:
self.useTagsAsDir = config.getboolean('Settings',
'useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print "useTagsAsDir = False"
haveError = True
try:
self.useBlacklistTags = config.getboolean(
'Settings', 'useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print "useBlacklistTags = False"
haveError = True
try:
self.useSuppressTags = config.getboolean(
'Settings', 'useSuppressTags')
except ValueError:
self.useSuppressTags = False
print "useSuppressTags = False"
haveError = True
try:
self.tagsLimit = config.getint('Settings', 'tagsLimit')
except ValueError:
self.tagsLimit = -1
print "tagsLimit = -1"
haveError = True
try:
self.useSSL = config.getboolean('Authentication', 'useSSL')
except ValueError:
self.useSSL = False
print "useSSL = False"
haveError = True
try:
self.writeImageInfo = config.getboolean(
'Settings', 'writeImageInfo')
except ValueError:
self.writeImageInfo = False
print "writeImageInfo = False"
haveError = True
try:
self.keepSignedIn = config.getint('Authentication',
'keepSignedIn')
except ValueError:
print "keepSignedIn = 0"
self.keepSignedIn = 0
haveError = True
try:
self.backupOldFile = config.getboolean('Settings',
'backupOldFile')
except ValueError:
self.backupOldFile = False
print "backupOldFile = False"
haveError = True
try:
self.logLevel = config.get('Settings', 'logLevel').upper()
if not self.logLevel in [
'CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO',
'DEBUG', 'NOTSET'
]:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print "logLevel = DEBUG"
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Settings', 'enableDump')
except ValueError:
print "enableDump = True"
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Settings', 'skipDumpFilter')
except ValueError:
print "skipDumpFilter = ''"
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean(
'Settings', 'dumpMediumPage')
except ValueError:
print "dumpMediumPage = False"
self.dumpMediumPage = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean(
'Settings', 'writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print "writeUgoiraInfo = False"
haveError = True
try:
self.createUgoira = config.getboolean('Settings',
'createUgoira')
except ValueError:
self.createUgoira = False
print "createUgoira = False"
haveError = True
try:
self.deleteZipFile = config.getboolean('Settings',
'deleteZipFile')
except ValueError:
self.deleteZipFile = False
print "deleteZipFile = False"
haveError = True
## except ConfigParser.NoOptionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
## except ConfigParser.NoSectionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
except:
print 'Error at loadConfig():', sys.exc_info()
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print 'Some configuration have invalid value, replacing with the default value.'
self.writeConfig(error=True)
print 'done.'
#-UI01B------write config
def writeConfig(self, error=False, path=None):
'''Backup old config if exist and write updated config.ini'''
print 'Writing config file...',
config = ConfigParser.RawConfigParser()
config.add_section('Settings')
config.add_section('Pixiv')
config.add_section('Authentication')
config.set('Settings', 'proxyAddress', self.proxyAddress)
config.set('Settings', 'useProxy', self.useProxy)
config.set('Settings', 'useragent', self.useragent)
config.set('Settings', 'debugHttp', self.debugHttp)
config.set('Settings', 'useRobots', self.useRobots)
config.set('Settings', 'filenameFormat', self.filenameFormat)
config.set('Settings', 'filenameMangaFormat', self.filenameMangaFormat)
config.set('Settings', 'timeout', self.timeout)
config.set('Settings', 'useList', self.useList)
config.set('Settings', 'processFromDb', self.processFromDb)
config.set('Settings', 'overwrite', self.overwrite)
config.set('Settings', 'tagsseparator', self.tagsSeparator)
config.set('Settings', 'daylastupdated', self.dayLastUpdated)
config.set('Settings', 'rootdirectory', self.rootDirectory)
config.set('Settings', 'retry', self.retry)
config.set('Settings', 'retrywait', self.retryWait)
config.set('Settings', 'createDownloadLists', self.createDownloadLists)
config.set('Settings', 'downloadListDirectory',
self.downloadListDirectory)
config.set('Settings', 'IrfanViewPath', self.IrfanViewPath)
config.set('Settings', 'startIrfanView', self.startIrfanView)
config.set('Settings', 'startIrfanSlide', self.startIrfanSlide)
config.set('Settings', 'alwaysCheckFileSize', self.alwaysCheckFileSize)
config.set('Settings', 'checkUpdatedLimit', self.checkUpdatedLimit)
config.set('Settings', 'downloadAvatar', self.downloadAvatar)
config.set('Settings', 'createMangaDir', self.createMangaDir)
config.set('Settings', 'useTagsAsDir', self.useTagsAsDir)
config.set('Settings', 'useBlacklistTags', self.useBlacklistTags)
config.set('Settings', 'useSuppressTags', self.useSuppressTags)
config.set('Settings', 'tagsLimit', self.tagsLimit)
config.set('Settings', 'writeImageInfo', self.writeImageInfo)
config.set('Settings', 'dateDiff', self.dateDiff)
config.set('Settings', 'backupOldFile', self.backupOldFile)
config.set('Settings', 'logLevel', self.logLevel)
config.set('Settings', 'enableDump', self.enableDump)
config.set('Settings', 'skipDumpFilter', self.skipDumpFilter)
config.set('Settings', 'dumpMediumPage', self.dumpMediumPage)
config.set('Settings', 'writeUgoiraInfo', self.writeUgoiraInfo)
config.set('Settings', 'createUgoira', self.createUgoira)
config.set('Settings', 'deleteZipFile', self.deleteZipFile)
config.set('Authentication', 'username', self.username)
config.set('Authentication', 'password', self.password)
config.set('Authentication', 'cookie', self.cookie)
config.set('Authentication', 'useSSL', self.useSSL)
config.set('Authentication', 'keepSignedIn', self.keepSignedIn)
config.set('Pixiv', 'numberOfPage', self.numberOfPage)
config.set('Pixiv', 'R18Mode', self.r18mode)
if path != None:
configlocation = path
else:
configlocation = 'config.ini'
try:
##with codecs.open('config.ini.bak', encoding = 'utf-8', mode = 'wb') as configfile:
with open(configlocation + '.tmp', 'w') as configfile:
config.write(configfile)
if os.path.exists(configlocation):
if error:
backupName = configlocation + '.error-' + str(
int(time.time()))
print "Backing up old config (error exist!) to " + backupName
shutil.move(configlocation, backupName)
else:
print "Backing up old config to config.ini.bak"
shutil.move(configlocation, configlocation + '.bak')
os.rename(configlocation + '.tmp', configlocation)
except:
self.__logger.exception('Error at writeConfig()')
raise
print 'done.'
def printConfig(self):
print 'Configuration: '
print ' [Authentication]'
print ' - username ='******' - password = '******' - cookie = ', self.cookie
print ' - useSSL = ', self.useSSL
print ' - keepSignedIn = ', self.keepSignedIn
print ' [Settings]'
print ' - filename_format =', self.filenameFormat
print ' - filename_manga_format =', self.filenameMangaFormat
print ' - useproxy =', self.useProxy
print ' - proxyaddress =', self.proxyAddress
print ' - debug_http =', self.debugHttp
print ' - use_robots =', self.useRobots
print ' - useragent =', self.useragent
print ' - overwrite =', self.overwrite
print ' - timeout =', self.timeout
print ' - useList =', self.useList
print ' - processFromDb =', self.processFromDb
print ' - tagsSeparator =', self.tagsSeparator
print ' - dayLastUpdated =', self.dayLastUpdated
print ' - rootDirectory =', self.rootDirectory
print ' - retry =', self.retry
print ' - retryWait =', self.retryWait
print ' - createDownloadLists =', self.createDownloadLists
print ' - downloadListDirectory =', self.downloadListDirectory
print ' - IrfanViewPath =', self.IrfanViewPath
print ' - startIrfanView =', self.startIrfanView
print ' - startIrfanSlide =', self.startIrfanSlide
print ' - alwaysCheckFileSize =', self.alwaysCheckFileSize
print ' - checkUpdatedLimit =', self.checkUpdatedLimit
print ' - downloadAvatar =', self.downloadAvatar
print ' - createMangaDir =', self.createMangaDir
print ' - useTagsAsDir =', self.useTagsAsDir
print ' - useBlacklistTags =', self.useBlacklistTags
print ' - useSuppressTags =', self.useSuppressTags
print ' - tagsLimit =', self.tagsLimit
print ' - writeImageInfo =', self.writeImageInfo
print ' - dateDiff =', self.dateDiff
print ' - backupOldFile =', self.backupOldFile
print ' - logLevel =', self.logLevel
print ' - enableDump =', self.enableDump
print ' - skipDumpFilter =', self.skipDumpFilter
print ' - dumpMediumPage =', self.dumpMediumPage
print ' - writeUgoiraInfo =', self.writeUgoiraInfo
print ' - createUgoira =', self.createUgoira
print ' - deleteZipFile =', self.deleteZipFile
print ' [Pixiv]'
print ' - numberOfPage =', self.numberOfPage
print ' - R18Mode =', self.r18mode
print ''
class TestPixivModel_WhiteCube(unittest.TestCase):
currPath = os.path.abspath('.')
PixivHelper.get_logger()
# @pytest.mark.xfail
# def testParseImage(self):
# p = open('./test/work_details_modal_whitecube.json', 'r')
# image = PixivImage(59521621, p.read())
# self.assertIsNotNone(image)
# image.PrintInfo()
# self.assertEqual(image.imageMode, "big")
#
# @pytest.mark.xfail
# def testParseManga(self):
# p = open('./test/work_details_modal_whitecube-manga.json', 'r')
# image = PixivImage(59532028, p.read())
# self.assertIsNotNone(image)
# image.PrintInfo()
# self.assertEqual(image.imageMode, "manga")
def testParseMemberError(self):
p = open('./test/ajax-error.json', 'r')
try:
PixivArtist(14095911, p.read())
self.fail("Exception expected.")
except PixivException as ex:
self.assertTrue(ex.errorCode == PixivException.OTHER_MEMBER_ERROR)
def testParseMemberImages(self):
p = open('./test/all-14095911.json', 'r')
member = PixivArtist(14095911, p.read(), False, 0, 24)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-14095911.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, False)
member.PrintInfo()
self.assertEqual(member.artistId, 14095911)
self.assertTrue(member.haveImages)
self.assertFalse(member.isLastPage)
self.assertEqual(member.artistToken, 'p199451')
self.assertGreater(member.totalImages, 70)
self.assertEqual(member.artistAvatar, "https://i.pximg.net/user-profile/img/2018/02/01/23/55/46/13768317_dc1474fef3dc982ade71c1b2dbf32d9e.png")
def testParseMemberImages2(self):
p = open('./test/all-26357.json', 'r')
member = PixivArtist(26357, p.read(), False, 0, 24)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-26357.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, False)
member.PrintInfo()
self.assertEqual(member.artistId, 26357)
self.assertTrue(member.haveImages)
self.assertFalse(member.isLastPage)
self.assertEqual(member.artistToken, 'yukimaruko')
self.assertGreater(member.totalImages, 7)
self.assertEqual(member.artistAvatar, "https://s.pximg.net/common/images/no_profile.png")
# https://www.pixiv.net/ajax/user/14095911/profile/all
def testParseMemberImagesLastPage(self):
p = open('./test/all-14095911.json', 'r')
member = PixivArtist(14095911, p.read(), False, 192, 48)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-14095911.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, False)
member.PrintInfo()
self.assertEqual(member.totalImages, 227)
self.assertTrue(member.isLastPage)
self.assertEqual(member.artistId, 14095911)
self.assertTrue(member.haveImages)
# /ajax/user/14095911/illustmanga/tag?tag=R-18&offset=0&limit=48
def testParseMemberImagesByTags(self):
p = open('./test/tag-R-18-14095911.json', 'r')
member = PixivArtist(14095911, p.read(), False, 0, 24)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-14095911.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, False)
member.PrintInfo()
self.assertEqual(member.artistId, 14095911)
self.assertTrue(member.haveImages)
self.assertFalse(member.isLastPage)
# /ajax/user/14095911/illustmanga/tag?tag=R-18&offset=48&limit=48
def testParseMemberImagesByTagsLastPage(self):
p = open('./test/tag-R-18-14095911-lastpage.json', 'r')
member = PixivArtist(14095911, p.read(), False, 96, 48)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-14095911.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, False)
member.PrintInfo()
self.assertEqual(member.totalImages, 129)
self.assertTrue(member.isLastPage)
self.assertEqual(member.artistId, 14095911)
self.assertTrue(member.haveImages)
def testParseMemberBookmarksByTags(self):
p = open('./test/bookmarks-1039353.json', 'r')
member = PixivArtist(1039353, p.read(), False, 0, 24)
self.assertIsNotNone(member)
p2 = open('./test/userdetail-1039353.json', 'r')
info = json.loads(p2.read())
member.ParseInfo(info, False, True)
member.PrintInfo()
self.assertEqual(member.artistId, 1039353)
self.assertTrue(member.haveImages)
self.assertFalse(member.isLastPage)
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename):
raise PixivException(
"File doesn't exists or no permission to read: " + filename,
errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION
)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 1
try:
for line in reader:
originalLine = line
##PixivHelper.safePrint("Processing: " + line)
if line.startswith('#') or len(line) < 1:
continue
if len(line.strip()) == 0:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(
dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
lineNo = lineNo + 1
originalLine = ""
except UnicodeDecodeError:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'
.format(originalLine, lineNo))
except:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
def ParseInfo(self, page):
self.imageUrls = list()
images = page.findAll(
"div", attrs={"class": "illust-zoom-in thumbnail-container"})
for image in images:
url = image["data-original-src"]
self.imageUrls.append(url)
if len(self.imageUrls) == 1:
self.imageMode = "big"
# TODO: handle ugoira
elif len(self.imageUrls) > 1:
self.imageMode = "manga"
# title/caption
self.imageTitle = page.findAll("div",
attrs={"class":
"title-container"})[0].text
descriptions = page.findAll(
"div", attrs={"class": "description-text ui-expander-target"})
if len(descriptions) > 0:
self.imageCaption = descriptions[0].text
# view count
self.jd_rtv = int(page.find(attrs={'class': 'react-count'}).text)
# like count
# react-count _clickable illust-bookmark-count-59521621 count like-count
self.jd_rtc = int(
page.find(
attrs={
'class':
"react-count _clickable illust-bookmark-count-{0} count like-count"
.format(self.imageId)
}).text)
# not available
self.jd_rtt = self.jd_rtc
# tags
self.imageTags = list()
# _tag-container tags illust-59521621
tagContainer = page.find("div",
attrs={
"class":
"_tag-container tags illust-{0}".format(
self.imageId)
})
# special node for R-18
r18Tag = page.findAll(attrs={'class': 'tag r-18'})
if r18Tag is not None:
self.imageTags.append("R-18")
tagLinks = tagContainer.findAll("a",
attrs={"class": re.compile(r"tag.*")})
for link in tagLinks:
self.imageTags.append(link["data-activity-tag_name"])
# date
self.worksDate = page.find("li", attrs={"class": "datetime"}).text
self.worksDateDateTime = PixivHelper.ParseDateTime(
self.worksDate, self.dateFormat)
# resolution
# tools
tools = page.findAll("li", attrs={"class": "tool"})
t = list()
for tool in tools:
t.append(tool.text)
if len(t) > 0:
self.worksTools = ", ".join(t)
def print_info(self):
works_per_page = 12
PixivHelper.safePrint('Manga Series Info')
PixivHelper.safePrint(f'Manga Series ID: {self.manga_series_id}')
PixivHelper.safePrint(f'Artist ID : {self.member_id}')
if self.artist is not None:
PixivHelper.safePrint(f'Artist Name : {self.artist.artistName}')
PixivHelper.safePrint(f'Title : {self.title}')
PixivHelper.safePrint(f'Description : {self.description}')
PixivHelper.safePrint(
f'Pages : {self.current_page} of {int(self.total_works/works_per_page)}'
)
PixivHelper.safePrint('Works :')
for (work_id, order) in self.pages_with_order:
PixivHelper.safePrint(f' - [{order}] {work_id}')
class TestPixivHelper(unittest.TestCase):
currPath = unicode(os.path.abspath('.'))
PixivHelper.GetLogger()
def testSanitizeFilename(self):
rootDir = '.'
filename = u'12345.jpg'
currentDir = os.path.abspath('.')
expected = currentDir + os.sep + filename
result = PixivHelper.sanitizeFilename(filename, rootDir)
self.assertEqual(result, expected)
self.assertTrue(len(result) < 255)
def testSanitizeFilename2(self):
rootDir = '.'
filename = u'12345.jpg'
currentDir = os.path.abspath('.')
expected = currentDir + os.sep + filename
result = PixivHelper.sanitizeFilename(filename, rootDir)
self.assertEqual(result, expected)
self.assertTrue(len(result) < 255)
@pytest.mark.xfail
def testCreateMangaFilename(self):
p = open('./test/test-image-manga.htm', 'r')
page = BeautifulSoup(p.read())
imageInfo = PixivImage(28820443, page)
imageInfo.imageCount = 100
page.decompose()
del page
# print(imageInfo.PrintInfo())
nameFormat = '%member_token% (%member_id%)\%urlFilename% %page_number% %works_date_only% %works_res% %works_tools% %title%'
expected = unicode(u'ffei (554800)\\28865189_p0 001 7-23-2012 複数枚投稿 2P Photoshop C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p0.jpg')
# print(result)
self.assertEqual(result, expected)
expected = unicode(u'ffei (554800)\\28865189_p14 015 7-23-2012 複数枚投稿 2P Photoshop C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p14.jpg')
# print(result)
self.assertEqual(result, expected)
expected = unicode(u'ffei (554800)\\28865189_p921 922 7-23-2012 複数枚投稿 2P Photoshop C82おまけ本 「沙耶は俺の嫁」サンプル.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img26/img/ffei/28865189_p921.jpg')
# print(result)
self.assertEqual(result, expected)
@pytest.mark.xfail
def testCreateFilenameUnicode(self):
p = open('./test/test-image-unicode.htm', 'r')
page = BeautifulSoup(p.read())
imageInfo = PixivImage(2493913, page)
page.decompose()
del page
nameFormat = '%member_token% (%member_id%)\%urlFilename% %works_date_only% %works_res% %works_tools% %title%'
expected = unicode(u'balzehn (267014)\\2493913 12-23-2008 852x1200 Photoshop SAI つけペン アラクネのいる日常２.jpg')
result = PixivHelper.makeFilename(nameFormat, imageInfo, artistInfo=None, tagsSeparator=' ', fileUrl='http://i2.pixiv.net/img16/img/balzehn/2493913.jpg')
# print(result)
self.assertEqual(result, expected)
def testcreateAvatarFilenameFormatNoSubfolderNoRootDir(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = ''
# change the config value
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%image_id% - %title%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
PixivHelper.setConfig(_config)
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, self.currPath + os.sep + u'folder.jpg')
def testcreateAvatarFilenameFormatWithSubfolderNoRootDir(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = ''
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%member_token% (%member_id%)' + os.sep + '%image_id% - %title% - %tags%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
PixivHelper.setConfig(_config)
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, self.currPath + os.sep + u'kirabara29 (1107124)' + os.sep + 'folder.jpg')
def testcreateAvatarFilenameFormatNoSubfolderWithRootDir3(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = os.path.abspath('.')
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%image_id% - %title%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, targetDir + os.sep + u'folder.jpg')
def testcreateAvatarFilenameFormatWithSubfolderWithRootDir4(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = os.path.abspath('.')
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%member_token% (%member_id%)' + os.sep + '%R-18%' + os.sep + '%image_id% - %title% - %tags%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, targetDir + os.sep + u'kirabara29 (1107124)' + os.sep + 'folder.jpg')
def testcreateAvatarFilenameFormatNoSubfolderWithCustomRootDir5(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = os.path.abspath(os.sep + 'images')
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%image_id% - %title%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, targetDir + os.sep + 'folder.jpg')
def testcreateAvatarFilenameFormatWithSubfolderWithCustomRootDir6(self):
p = open('./test/test-helper-avatar-name.htm', 'r')
page = BeautifulSoup(p.read())
artist = PixivArtist(mid=1107124, page=page)
targetDir = os.path.abspath(os.sep + 'images')
_config = PixivConfig.PixivConfig()
_config.avatarNameFormat = ''
_config.filenameFormat = '%member_token% (%member_id%)' + os.sep + '%R-18%' + os.sep + '%image_id% - %title% - %tags%'
_config.tagsSeparator = ' '
_config.tagsLimit = 0
filename = PixivHelper.createAvatarFilename(artist, targetDir)
self.assertEqual(filename, targetDir + os.sep + 'kirabara29 (1107124)' + os.sep + 'folder.jpg')
def testParseLoginError(self):
p = open('./test/test-login-error.htm', 'r')
page = BeautifulSoup(p.read())
r = page.findAll('span', attrs={'class': 'error'})
self.assertTrue(len(r) > 0)
self.assertEqual(u'Please ensure your pixiv ID, email address and password is entered correctly.', r[0].string)
def testParseLoginForm(self):
p = open('./test/test-login-form.html', 'r')
page = BeautifulSoup(p.read())
r = page.findAll('form', attrs={'action': '/login.php'})
# print(r)
self.assertTrue(len(r) > 0)
def process_member(caller,
config,
member_id,
user_dir='',
page=1,
end_page=0,
bookmark=False,
tags=None,
title_prefix="",
bookmark_count=None,
notifier=None):
# caller function/method
# TODO: ideally to be removed or passed as argument
db = caller.__dbManager__
# np = caller.np
# np_is_valid = caller.np_is_valid
if notifier is None:
notifier = PixivHelper.dummy_notifier
list_page = None
msg = Fore.YELLOW + Style.BRIGHT + f'Processing Member Id: {member_id}' + Style.RESET_ALL
PixivHelper.print_and_log('info', msg)
notifier(type="MEMBER", message=msg)
if page != 1:
PixivHelper.print_and_log('info', 'Start Page: ' + str(page))
if end_page != 0:
PixivHelper.print_and_log('info', 'End Page: ' + str(end_page))
if config.numberOfPage != 0:
PixivHelper.print_and_log(
'info', 'Number of page setting will be ignored')
elif config.numberOfPage != 0:
PixivHelper.print_and_log(
'info', f'End Page from config: {config.numberOfPage}')
# calculate the offset for display properties
offset = 48 # new offset for AJAX call
offset_start = (page - 1) * offset
offset_stop = end_page * offset
try:
no_of_images = 1
is_avatar_downloaded = False
flag = True
updated_limit_count = 0
image_id = -1
while flag:
PixivHelper.print_and_log(None, 'Page ', page)
caller.set_console_title(
f"{title_prefix}MemberId: {member_id} Page: {page}")
# Try to get the member page
while True:
try:
(artist, list_page
) = PixivBrowserFactory.getBrowser().getMemberPage(
member_id,
page,
bookmark,
tags,
r18mode=config.r18mode,
throw_empty_error=True)
break
except PixivException as ex:
caller.ERROR_CODE = ex.errorCode
PixivHelper.print_and_log(
'info', f'Member ID ({member_id}): {ex}')
if ex.errorCode == PixivException.NO_IMAGES:
pass
else:
if list_page is None:
list_page = ex.htmlPage
if list_page is not None:
PixivHelper.dump_html(
f"Dump for {member_id} Error Code {ex.errorCode}.html",
list_page)
if ex.errorCode == PixivException.USER_ID_NOT_EXISTS or ex.errorCode == PixivException.USER_ID_SUSPENDED:
db.setIsDeletedFlagForMemberId(int(member_id))
PixivHelper.print_and_log(
'info',
f'Set IsDeleted for MemberId: {member_id} not exist.'
)
# db.deleteMemberByMemberId(member_id)
# PixivHelper.printAndLog('info', 'Deleting MemberId: ' + str(member_id) + ' not exist.')
if ex.errorCode == PixivException.OTHER_MEMBER_ERROR:
PixivHelper.print_and_log(None, ex.message)
caller.__errorList.append(
dict(type="Member",
id=str(member_id),
message=ex.message,
exception=ex))
return
except AttributeError:
# Possible layout changes, try to dump the file below
raise
except BaseException:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value,
exc_traceback)
PixivHelper.print_and_log(
'error',
f'Error at processing Artist Info: {sys.exc_info()}')
PixivHelper.print_and_log(None,
f'Member Name : {artist.artistName}')
PixivHelper.print_and_log(None,
f'Member Avatar: {artist.artistAvatar}')
PixivHelper.print_and_log(None,
f'Member Token : {artist.artistToken}')
PixivHelper.print_and_log(
None, f'Member Background : {artist.artistBackground}')
print_offset_stop = offset_stop if offset_stop < artist.totalImages and offset_stop != 0 else artist.totalImages
PixivHelper.print_and_log(
None,
f'Processing images from {offset_start + 1} to {print_offset_stop} of {artist.totalImages}'
)
if not is_avatar_downloaded and config.downloadAvatar:
if user_dir == '':
target_dir = config.rootDirectory
else:
target_dir = user_dir
(filename_avatar,
filename_bg) = PixivHelper.create_avabg_filename(
artist, target_dir, config)
if not caller.DEBUG_SKIP_PROCESS_IMAGE:
if artist.artistAvatar.find('no_profile') == -1:
PixivDownloadHandler.download_image(
caller,
artist.artistAvatar,
filename_avatar,
"https://www.pixiv.net/",
config.overwrite,
config.retry,
config.backupOldFile,
notifier=notifier)
# Issue #508
if artist.artistBackground is not None and artist.artistBackground.startswith(
"http"):
PixivDownloadHandler.download_image(
caller,
artist.artistBackground,
filename_bg,
"https://www.pixiv.net/",
config.overwrite,
config.retry,
config.backupOldFile,
notifier=notifier)
is_avatar_downloaded = True
if config.autoAddMember:
db.insertNewMember(int(member_id),
member_token=artist.artistToken)
db.updateMemberName(member_id, artist.artistName,
artist.artistToken)
if not artist.haveImages:
PixivHelper.print_and_log('info',
f"No image found for: {member_id}")
db.updateLastDownloadDate(member_id)
flag = False
continue
result = PixivConstant.PIXIVUTIL_NOT_OK
for image_id in artist.imageList:
PixivHelper.print_and_log(None, f'#{no_of_images}')
retry_count = 0
while True:
try:
if artist.totalImages > 0:
# PixivHelper.safePrint("Total Images = " + str(artist.totalImages))
total_image_page_count = artist.totalImages
if (offset_stop > 0
and offset_stop < total_image_page_count):
total_image_page_count = offset_stop
total_image_page_count = total_image_page_count - offset_start
# PixivHelper.safePrint("Total Images Offset = " + str(total_image_page_count))
else:
total_image_page_count = (
(page - 1) * 20) + len(artist.imageList)
title_prefix_img = f"{title_prefix}MemberId: {member_id} Page: {page} Post {no_of_images}+{updated_limit_count} of {total_image_page_count}"
if not caller.DEBUG_SKIP_PROCESS_IMAGE:
result = PixivImageHandler.process_image(
caller,
config,
artist,
image_id,
user_dir,
bookmark,
title_prefix=title_prefix_img,
bookmark_count=bookmark_count,
notifier=notifier)
break
except KeyboardInterrupt:
result = PixivConstant.PIXIVUTIL_KEYBOARD_INTERRUPT
break
except BaseException:
if retry_count > config.retry:
PixivHelper.print_and_log(
'error', f"Giving up image_id: {image_id}")
return
retry_count = retry_count + 1
PixivHelper.print_and_log(
None,
f"Stuff happened, trying again after 2 second ({retry_count})"
)
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value,
exc_traceback)
PixivHelper.print_and_log(
"error",
f"Error at process_member(): {sys.exc_info()} Member Id: {member_id}"
)
PixivHelper.print_delay(2)
if result in (PixivConstant.PIXIVUTIL_SKIP_DUPLICATE,
PixivConstant.PIXIVUTIL_SKIP_LOCAL_LARGER,
PixivConstant.PIXIVUTIL_SKIP_DUPLICATE_NO_WAIT):
updated_limit_count = updated_limit_count + 1
if config.checkUpdatedLimit != 0 and updated_limit_count >= config.checkUpdatedLimit:
PixivHelper.safePrint(f"Skipping member: {member_id}")
db.updateLastDownloadDate(member_id)
PixivBrowserFactory.getBrowser(
config=config).clear_history()
return
gc.collect()
continue
if result == PixivConstant.PIXIVUTIL_KEYBOARD_INTERRUPT:
choice = input(
"Keyboard Interrupt detected, continue to next image (Y/N)"
).rstrip("\r")
if choice.upper() == 'N':
PixivHelper.print_and_log(
"info", f"Member: {member_id}, processing aborted")
flag = False
break
else:
continue
# return code from process image
if result == PixivConstant.PIXIVUTIL_SKIP_OLDER:
PixivHelper.print_and_log(
"info",
"Reached older images, skippin to next member.")
db.updateLastDownloadDate(member_id)
flag = False
break
no_of_images = no_of_images + 1
PixivHelper.wait(result, config)
if artist.isLastPage:
db.updateLastDownloadDate(member_id)
PixivHelper.print_and_log(None, "Last Page")
flag = False
page = page + 1
# page limit checking
if end_page > 0 and page > end_page:
PixivHelper.print_and_log(
None,
f"Page limit reached (from endPage limit ={end_page})")
db.updateLastDownloadDate(member_id)
flag = False
elif config.numberOfPage > 0 and page > config.numberOfPage:
PixivHelper.print_and_log(
None,
f"Page limit reached (from config ={config.numberOfPage})")
flag = False
del artist
del list_page
PixivBrowserFactory.getBrowser(config=config).clear_history()
gc.collect()
log_message = ""
if int(image_id) > 0:
db.updateLastDownloadedImage(member_id, image_id)
log_message = f'last image_id: {image_id}'
else:
log_message = 'no images were found.'
PixivHelper.print_and_log(
"info", f"Member_id: {member_id} completed: {log_message}")
except KeyboardInterrupt:
raise
except Exception:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
PixivHelper.print_and_log(
'error', f'Error at process_member(): {sys.exc_info()}')
try:
if list_page is not None:
dump_filename = f'Error page for member {member_id} at page {page}.html'
PixivHelper.dump_html(dump_filename, list_page)
PixivHelper.print_and_log('error',
f"Dumping html to: {dump_filename}")
except BaseException:
PixivHelper.print_and_log(
'error', f'Cannot dump page for member_id: {member_id}')
raise
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename):
raise PixivException(
"File doesn't exists or no permission to read: " + filename,
errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION
)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 1
try:
for line in reader:
originalLine = line
#PixivHelper.safePrint("Processing: " + line)
if line.startswith('#') or len(line) < 1:
continue
if len(line.strip()) == 0:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
if items[0].startswith("http"):
# handle urls:
# http://www.pixiv.net/member_illust.php?id=<member_id>
# http://www.pixiv.net/member.php?id=<member_id>
parsed = urlparse.urlparse(items[0])
if parsed.path == "/member.php" or parsed.path == "/member_illust.php":
query_str = urlparse.parse_qs(parsed.query)
if query_str.has_key("id"):
member_id = int(query_str["id"][0])
else:
PixivHelper.printAndLog(
'error', "Cannot detect member id from url: " +
items[0])
continue
else:
PixivHelper.printAndLog(
'error', "Unsupported url detected: " + items[0])
continue
else:
# handle member id directly
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(
dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
lineNo = lineNo + 1
originalLine = ""
except UnicodeDecodeError:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'
.format(originalLine, lineNo))
except:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
