def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename) :
raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 1
try:
for line in reader:
originalLine = line
##PixivHelper.safePrint("Processing: " + line)
if line.startswith('#') or len(line) < 1:
continue
if len(line.strip()) == 0:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(" ", 1)
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir != None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
l.append(listItem)
lineNo = lineNo + 1
originalLine = ""
except UnicodeDecodeError:
PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'.format(originalLine, lineNo))
except:
PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
def parseTagsList(filename):
'''read tags.txt and return the tags list'''
l = list()
if not os.path.exists(filename) :
raise PixivException("File doesn't exists or no permission to read: " + filename, FILE_NOT_EXISTS_OR_NO_READ_PERMISSION)
reader = PixivHelper.OpenTextFile(filename)
for line in reader:
if line.startswith('#') or len(line) < 1:
continue
line = line.strip()
if len(line) > 0 :
l.append(PixivHelper.toUnicode(line))
reader.close()
return l
def loadConfig(self, path=None):
''' new settings must be added on the last'''
if path is not None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print('Reading', self.configFileLocation, '...')
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication', 'username')
self.password = config.get('Authentication', 'password')
self.cookie = config.get('Authentication', 'cookie')
self.tagsSeparator = PixivHelper.toUnicode(config.get('Filename', 'tagsseparator'), encoding=sys.stdin.encoding)
self.rootDirectory = os.path.expanduser(PixivHelper.toUnicode(config.get('Settings', 'rootdirectory'), encoding=sys.stdin.encoding))
try:
self.IrfanViewPath = os.path.expanduser(
PixivHelper.toUnicode(config.get('IrfanView', 'IrfanViewPath'), encoding=sys.stdin.encoding))
self.downloadListDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings', 'downloadListDirectory'), encoding=sys.stdin.encoding))
except BaseException:
pass
try:
self.processFromDb = config.getboolean('Settings', 'processfromdb')
except ValueError:
print("processFromDb = True")
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings', 'daylastupdated')
except ValueError:
print("dayLastUpdated = 7")
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings', 'datediff')
except ValueError:
print("dateDiff = 0")
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Network', 'proxyaddress')
except ValueError:
print("proxyAddress = ''")
self.proxyAddress = ''
haveError = True
self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress}
try:
self.useProxy = config.getboolean('Network', 'useproxy')
except ValueError:
print("useProxy = False")
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings', 'uselist')
except ValueError:
print("useList = False")
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv', 'r18mode')
except ValueError:
print("r18mode = False")
self.r18mode = False
haveError = True
_useragent = config.get('Network', 'useragent')
if _useragent is not None:
self.useragent = _useragent
_filenameFormat = config.get('Filename', 'filenameformat')
_filenameFormat = PixivHelper.toUnicode(_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat is not None and len(_filenameFormat) > 0:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Filename', 'filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat is not None and len(_filenameMangaFormat) > 0:
# check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find('%page_index%') == -1 and _filenameMangaFormat.find('%page_number%') == -1:
print('No page identifier, appending %page_index% to the filename manga format.')
_filenameMangaFormat = _filenameMangaFormat + unicode(' %page_index%')
print("_filenameMangaFormat =", _filenameMangaFormat)
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
_filenameInfoFormat = config.get('Filename', 'filenameinfoformat')
_filenameInfoFormat = PixivHelper.toUnicode(_filenameInfoFormat, encoding=sys.stdin.encoding)
if _filenameInfoFormat is not None and len(_filenameInfoFormat) > 0:
self.filenameInfoFormat = _filenameInfoFormat
try:
self.debugHttp = config.getboolean('Debug', 'debughttp')
except ValueError:
self.debugHttp = False
print("debugHttp = False")
haveError = True
try:
self.useRobots = config.getboolean('Network', 'userobots')
except ValueError:
self.useRobots = False
print("useRobots = False")
haveError = True
try:
self.overwrite = config.getboolean('Settings', 'overwrite')
except ValueError:
print("overwrite = False")
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean('Filename', 'createMangaDir')
except ValueError:
print("createMangaDir = False")
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Network', 'timeout')
except ValueError:
print("timeout = 60")
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Network', 'retry')
except ValueError:
print("retry = 3")
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Network', 'retrywait')
except ValueError:
print("retryWait = 5")
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv', 'numberofpage')
except ValueError:
self.numberOfPage = 0
print("numberOfPage = 0")
haveError = True
try:
self.createDownloadLists = config.getboolean('IrfanView', 'createDownloadLists')
except ValueError:
self.createDownloadLists = False
print("createDownloadLists = False")
haveError = True
try:
self.startIrfanView = config.getboolean('IrfanView', 'startIrfanView')
except ValueError:
self.startIrfanView = False
print("startIrfanView = False")
haveError = True
try:
self.startIrfanSlide = config.getboolean('IrfanView', 'startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print("startIrfanSlide = False")
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean('Settings', 'alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print("alwaysCheckFileSize = False")
haveError = True
try:
self.downloadAvatar = config.getboolean('Settings', 'downloadAvatar')
except ValueError:
self.downloadAvatar = False
print("downloadAvatar = False")
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings', 'checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print("checkUpdatedLimit = 0")
haveError = True
try:
self.useTagsAsDir = config.getboolean('Filename', 'useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print("useTagsAsDir = False")
haveError = True
try:
self.useBlacklistTags = config.getboolean('Settings', 'useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print("useBlacklistTags = False")
haveError = True
try:
self.useSuppressTags = config.getboolean('Settings', 'useSuppressTags')
except ValueError:
self.useSuppressTags = False
print("useSuppressTags = False")
haveError = True
try:
self.tagsLimit = config.getint('Settings', 'tagsLimit')
except ValueError:
self.tagsLimit = -1
print("tagsLimit = -1")
haveError = True
try:
self.writeImageInfo = config.getboolean('Settings', 'writeImageInfo')
except ValueError:
self.writeImageInfo = False
print("writeImageInfo = False")
haveError = True
try:
self.backupOldFile = config.getboolean('Settings', 'backupOldFile')
except ValueError:
self.backupOldFile = False
print("backupOldFile = False")
haveError = True
try:
self.logLevel = config.get('Debug', 'logLevel').upper()
if self.logLevel not in ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print("logLevel = DEBUG")
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Debug', 'enableDump')
except ValueError:
print("enableDump = True")
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Debug', 'skipDumpFilter')
except ValueError:
print("skipDumpFilter = ''")
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean('Debug', 'dumpMediumPage')
except ValueError:
print("dumpMediumPage = False")
self.dumpMediumPage = False
haveError = True
try:
self.enableInfiniteLoop = config.getboolean('Settings', 'enableInfiniteLoop')
except ValueError:
self.enableInfiniteLoop = False
print("enableInfiniteLoop = False")
haveError = True
try:
self.dumpTagSearchPage = config.getboolean('Debug', 'dumpTagSearchPage')
except ValueError:
self.dumpTagSearchPage = False
print("dumpTagSearchPage = False")
haveError = True
try:
self.dateFormat = config.get('Pixiv', 'dateFormat')
except ValueError:
print("dateFormat = ''")
self.dateFormat = ''
haveError = True
try:
self.verifyImage = config.getboolean('Settings', 'verifyImage')
except ValueError:
print("verifyImage = False")
self.verifyImage = False
haveError = True
try:
self.writeUrlInDescription = config.getboolean('Settings', 'writeUrlInDescription')
except ValueError:
print("writeUrlInDescription = False")
self.writeUrlInDescription = False
haveError = True
try:
self.urlBlacklistRegex = config.get('Settings', 'urlBlacklistRegex')
except ValueError:
print("urlBlacklistRegex = ")
self.urlBlacklistRegex = ""
haveError = True
try:
self.urlDumpFilename = config.get('Filename', 'urlDumpFilename')
except ValueError:
print("urlDumpFilename = url_list_%Y%m%d")
self.urlDumpFilename = "url_list_%Y%m%d"
haveError = True
try:
self.dbPath = config.get('Settings', 'dbPath')
except ValueError:
print("dbPath = ''")
self.dbPath = ''
haveError = True
try:
self.useBlacklistMembers = config.getboolean('Settings', 'useBlacklistMembers')
except ValueError:
print("useBlacklistMembers = False")
self.useBlacklistMembers = False
haveError = True
try:
self.avatarNameFormat = config.get('Filename', 'avatarNameFormat')
self.avatarNameFormat = PixivHelper.toUnicode(self.avatarNameFormat, encoding=sys.stdin.encoding)
except ValueError:
print("avatarNameFormat = ")
self.avatarNameFormat = ""
haveError = True
try:
self.writeImageJSON = config.getboolean('Settings', 'writeImageJSON')
except ValueError:
self.writeImageJSON = False
print("writeImageJSON = False")
haveError = True
try:
self.downloadDelay = config.getint('Network', 'downloadDelay')
except ValueError:
self.downloadDelay = 2
print("downloadDelay = 2")
haveError = True
try:
self.ffmpeg = config.get('FFmpeg', 'ffmpeg')
except ValueError:
print("ffmpeg = 'ffmpeg'")
self.ffmpeg = 'ffmpeg'
haveError = True
try:
self.ffmpegCodec = config.get('FFmpeg', 'ffmpegCodec')
except ValueError:
print("ffmpegCodec = 'libvpx-vp9'")
self.ffmpegCodec = 'libvpx-vp9'
haveError = True
try:
self.ffmpegParam = config.get('FFmpeg', 'ffmpegParam')
except ValueError:
print("ffmpegParam = '-lossless 1'")
self.ffmpegParam = '-lossless 1'
haveError = True
try:
self.setLastModified = config.getboolean('Settings', 'setLastModified')
except ValueError:
print("setLastModified = True")
self.setLastModified = True
haveError = True
try:
self.useLocalTimezone = config.getboolean('Settings', 'useLocalTimezone')
except ValueError:
print("useLocalTimezone = False")
self.useLocalTimezone = False
haveError = True
try:
self.checkNewVersion = config.getboolean('Network', 'checkNewVersion')
except ValueError:
print("checkNewVersion = True")
self.checkNewVersion = True
haveError = True
try:
self.webpCodec = config.get('FFmpeg', 'webpCodec')
except ValueError:
print("webpCodec = 'libwebp'")
self.webpCodec = 'libwebp'
haveError = True
try:
self.webpParam = config.get('FFmpeg', 'webpParam')
except ValueError:
print("webpParam = '-lossless 0 -q:v 90'")
self.webpParam = '-lossless 0 -q:v 90'
haveError = True
try:
self.createWebp = config.getboolean('Ugoira', 'createWebp')
except ValueError:
print("createWebp = False")
self.createWebp = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean('Ugoira', 'writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print("writeUgoiraInfo = False")
haveError = True
try:
self.createUgoira = config.getboolean('Ugoira', 'createUgoira')
except ValueError:
self.createUgoira = False
print("createUgoira = False")
haveError = True
try:
self.deleteZipFile = config.getboolean('Ugoira', 'deleteZipFile')
except ValueError:
self.deleteZipFile = False
print("deleteZipFile = False")
haveError = True
try:
self.createGif = config.getboolean('Ugoira', 'createGif')
except ValueError:
print("createGif = False")
self.createGif = False
haveError = True
try:
self.createApng = config.getboolean('Ugoira', 'createApng')
except ValueError:
print("createApng = False")
self.createApng = False
haveError = True
try:
self.deleteUgoira = config.getboolean('Ugoira', 'deleteUgoira')
except ValueError:
print("deleteUgoira = False")
self.deleteUgoira = False
haveError = True
try:
self.createWebm = config.getboolean('Ugoira', 'createWebm')
except ValueError:
print("createWebm = False")
self.createWebm = False
haveError = True
try:
self.refresh_token = config.get('Authentication', 'refresh_token')
except ValueError:
print("refresh_token = ''")
self.refresh_token = None
haveError = True
try:
self.enableSSLVerification = config.getboolean('Network', 'enableSSLVerification')
except ValueError:
print("enableSSLVerification = False")
self.enableSSLVerification = False
haveError = True
except BaseException:
print('Error at loadConfig():', sys.exc_info())
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print('Some configuration have invalid value, replacing with the default value.')
self.writeConfig(error=True)
print('done.')
def loadConfig(self, path=None):
''' new settings must be added on the last'''
if path != None:
self.configFileLocation = path
else:
self.configFileLocation = script_path + os.sep + 'config.ini'
print 'Reading', self.configFileLocation, '...'
haveError = False
config = ConfigParser.RawConfigParser()
try:
config.readfp(PixivHelper.OpenTextFile(self.configFileLocation))
self.username = config.get('Authentication','username')
self.password = config.get('Authentication','password')
self.cookie = config.get('Authentication','cookie')
self.tagsSeparator = PixivHelper.toUnicode(config.get('Settings','tagsseparator'), encoding=sys.stdin.encoding)
self.rootDirectory = os.path.expanduser(PixivHelper.toUnicode(config.get('Settings','rootdirectory'), encoding=sys.stdin.encoding))
try:
self.IrfanViewPath = os.path.expanduser(
PixivHelper.toUnicode(config.get('IrfanView','IrfanViewPath'), encoding=sys.stdin.encoding))
self.downloadListDirectory = os.path.expanduser(
PixivHelper.toUnicode(config.get('Settings','downloadListDirectory'), encoding=sys.stdin.encoding))
except:
pass
try:
self.processFromDb = config.getboolean('Settings','processfromdb')
except ValueError:
print "processFromDb = True"
self.processFromDb = True
haveError = True
try:
self.dayLastUpdated = config.getint('Settings','daylastupdated')
except ValueError:
print "dayLastUpdated = 7"
self.dayLastUpdated = 7
haveError = True
try:
self.dateDiff = config.getint('Settings','datediff')
except ValueError:
print "dateDiff = 0"
self.dateDiff = 0
haveError = True
try:
self.proxyAddress = config.get('Network','proxyaddress')
except ValueError:
print "proxyAddress = ''"
self.proxyAddress = ''
haveError = True
self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress}
try:
self.useProxy = config.getboolean('Network','useproxy')
except ValueError:
print "useProxy = False"
self.useProxy = False
haveError = True
try:
self.useList = config.getboolean('Settings','uselist')
except ValueError:
print "useList = False"
self.useList = False
haveError = True
try:
self.r18mode = config.getboolean('Pixiv','r18mode')
except ValueError:
print "r18mode = False"
self.r18mode = False
haveError = True
_useragent = config.get('Network','useragent')
if _useragent != None:
self.useragent = _useragent
_filenameFormat = config.get('Settings','filenameformat')
_filenameFormat = PixivHelper.toUnicode(_filenameFormat, encoding=sys.stdin.encoding)
if _filenameFormat != None:
self.filenameFormat = _filenameFormat
_filenameMangaFormat = config.get('Settings','filenamemangaformat')
_filenameMangaFormat = PixivHelper.toUnicode(_filenameMangaFormat, encoding=sys.stdin.encoding)
if _filenameMangaFormat != None:
## check if the filename format have page identifier if not using %urlFilename%
if _filenameMangaFormat.find('%urlFilename%') == -1:
if _filenameMangaFormat.find('%page_index%') == -1 and _filenameMangaFormat.find('%page_number%') == -1:
print 'No page identifier, appending %page_index% to the filename manga format.'
_filenameMangaFormat = _filenameMangaFormat + unicode(' %page_index%')
print "_filenameMangaFormat =", _filenameMangaFormat
haveError = True
self.filenameMangaFormat = _filenameMangaFormat
try:
self.debugHttp = config.getboolean('Debug','debughttp')
except ValueError:
self.debugHttp = False
print "debugHttp = False"
haveError = True
try:
self.useRobots = config.getboolean('Network','userobots')
except ValueError:
self.useRobots = False
print "useRobots = False"
haveError = True
try:
self.overwrite = config.getboolean('Settings','overwrite')
except ValueError:
print "overwrite = False"
self.overwrite = False
haveError = True
try:
self.createMangaDir = config.getboolean('Settings','createMangaDir')
except ValueError:
print "createMangaDir = False"
self.createMangaDir = False
haveError = True
try:
self.timeout = config.getint('Network','timeout')
except ValueError:
print "timeout = 60"
self.timeout = 60
haveError = True
try:
self.retry = config.getint('Network','retry')
except ValueError:
print "retry = 3"
self.retry = 3
haveError = True
try:
self.retryWait = config.getint('Network','retrywait')
except ValueError:
print "retryWait = 5"
self.retryWait = 5
haveError = True
try:
self.numberOfPage = config.getint('Pixiv','numberofpage')
except ValueError:
self.numberOfPage = 0
print "numberOfPage = 0"
haveError = True
try:
self.createDownloadLists = config.getboolean('IrfanView','createDownloadLists')
except ValueError:
self.createDownloadLists = False
print "createDownloadLists = False"
haveError = True
try:
self.startIrfanView = config.getboolean('IrfanView','startIrfanView')
except ValueError:
self.startIrfanView = False
print "startIrfanView = False"
haveError = True
try:
self.startIrfanSlide = config.getboolean('IrfanView','startIrfanSlide')
except ValueError:
self.startIrfanSlide = False
print "startIrfanSlide = False"
haveError = True
try:
self.alwaysCheckFileSize = config.getboolean('Settings','alwaysCheckFileSize')
except ValueError:
self.alwaysCheckFileSize = False
print "alwaysCheckFileSize = False"
haveError = True
try:
self.downloadAvatar = config.getboolean('Settings','downloadAvatar')
except ValueError:
self.downloadAvatar = False
print "downloadAvatar = False"
haveError = True
try:
self.checkUpdatedLimit = config.getint('Settings','checkUpdatedLimit')
except ValueError:
self.checkUpdatedLimit = 0
print "checkUpdatedLimit = 0"
haveError = True
try:
self.useTagsAsDir = config.getboolean('Settings','useTagsAsDir')
except ValueError:
self.useTagsAsDir = False
print "useTagsAsDir = False"
haveError = True
try:
self.useBlacklistTags = config.getboolean('Settings','useBlacklistTags')
except ValueError:
self.useBlacklistTags = False
print "useBlacklistTags = False"
haveError = True
try:
self.useSuppressTags = config.getboolean('Settings','useSuppressTags')
except ValueError:
self.useSuppressTags = False
print "useSuppressTags = False"
haveError = True
try:
self.tagsLimit = config.getint('Settings','tagsLimit')
except ValueError:
self.tagsLimit = -1
print "tagsLimit = -1"
haveError = True
try:
self.useSSL = config.getboolean('Authentication','useSSL')
except ValueError:
self.useSSL = False
print "useSSL = False"
haveError = True
try:
self.writeImageInfo = config.getboolean('Settings','writeImageInfo')
except ValueError:
self.writeImageInfo = False
print "writeImageInfo = False"
haveError = True
try:
self.keepSignedIn = config.getint('Authentication','keepSignedIn')
except ValueError:
print "keepSignedIn = 0"
self.keepSignedIn = 0
haveError = True
try:
self.backupOldFile = config.getboolean('Settings','backupOldFile')
except ValueError:
self.backupOldFile = False
print "backupOldFile = False"
haveError = True
try:
self.logLevel = config.get('Debug','logLevel').upper()
if not self.logLevel in ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']:
raise ValueError("Value not in list: " + self.logLevel)
except ValueError:
print "logLevel = DEBUG"
self.logLevel = 'DEBUG'
haveError = True
try:
self.enableDump = config.getboolean('Debug','enableDump')
except ValueError:
print "enableDump = True"
self.enableDump = True
haveError = True
try:
self.skipDumpFilter = config.get('Debug','skipDumpFilter')
except ValueError:
print "skipDumpFilter = ''"
self.skipDumpFilter = ''
haveError = True
try:
self.dumpMediumPage = config.getboolean('Debug','dumpMediumPage')
except ValueError:
print "dumpMediumPage = False"
self.dumpMediumPage = False
haveError = True
try:
self.writeUgoiraInfo = config.getboolean('Settings','writeUgoiraInfo')
except ValueError:
self.writeUgoiraInfo = False
print "writeUgoiraInfo = False"
haveError = True
try:
self.createUgoira = config.getboolean('Settings','createUgoira')
except ValueError:
self.createUgoira = False
print "createUgoira = False"
haveError = True
try:
self.deleteZipFile = config.getboolean('Settings','deleteZipFile')
except ValueError:
self.deleteZipFile = False
print "deleteZipFile = False"
haveError = True
try:
self.enableInfiniteLoop = config.getboolean('Settings','enableInfiniteLoop')
except ValueError:
self.enableInfiniteLoop = False
print "enableInfiniteLoop = False"
haveError = True
try:
self.dumpTagSearchPage = config.getboolean('Debug','dumpTagSearchPage')
except ValueError:
self.dumpTagSearchPage = False
print "dumpTagSearchPage = False"
haveError = True
try:
self.dateFormat = config.get('Pixiv','dateFormat')
except ValueError:
print "dateFormat = ''"
self.dateFormat = ''
haveError = True
try:
self.verifyImage = config.getboolean('Settings','verifyImage')
except ValueError:
print "verifyImage = False"
self.verifyImage = False
haveError = True
try:
self.writeUrlInDescription = config.getboolean('Settings','writeUrlInDescription')
except ValueError:
print "writeUrlInDescription = False"
self.writeUrlInDescription = False
haveError = True
## except ConfigParser.NoOptionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
## except ConfigParser.NoSectionError:
## print 'Error at loadConfig():',sys.exc_info()
## print 'Failed to read configuration.'
## self.writeConfig()
except:
print 'Error at loadConfig():',sys.exc_info()
self.__logger.exception('Error at loadConfig()')
haveError = True
if haveError:
print 'Some configuration have invalid value, replacing with the default value.'
self.writeConfig(error=True)
print 'done.'
def parseList(filename, rootDir=None):
'''read list.txt and return the list of PixivListItem'''
l = list()
if not os.path.exists(filename):
raise PixivException(
"File doesn't exists or no permission to read: " + filename,
errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION
)
reader = PixivHelper.OpenTextFile(filename)
lineNo = 1
try:
for line in reader:
originalLine = line
# PixivHelper.safePrint("Processing: " + line)
if line.startswith('#') or len(line) < 1:
continue
if len(line.strip()) == 0:
continue
line = PixivHelper.toUnicode(line)
line = line.strip()
items = line.split(None, 1)
if items[0].startswith("http"):
# handle urls:
# http://www.pixiv.net/member_illust.php?id=<member_id>
# http://www.pixiv.net/member.php?id=<member_id>
parsed = urlparse.urlparse(items[0])
if parsed.path == "/member.php" or parsed.path == "/member_illust.php":
query_str = urlparse.parse_qs(parsed.query)
if query_str.has_key("id"):
member_id = int(query_str["id"][0])
else:
PixivHelper.printAndLog(
'error', "Cannot detect member id from url: " +
items[0])
continue
else:
PixivHelper.printAndLog(
'error', "Unsupported url detected: " + items[0])
continue
else:
# handle member id directly
member_id = int(items[0])
path = ""
if len(items) > 1:
path = items[1].strip()
path = path.replace('\"', '')
if rootDir is not None:
path = path.replace('%root%', rootDir)
else:
path = path.replace('%root%', '')
path = os.path.abspath(path)
# have drive letter
if re.match(r'[a-zA-Z]:', path):
dirpath = path.split(os.sep, 1)
dirpath[1] = PixivHelper.sanitizeFilename(
dirpath[1], None)
path = os.sep.join(dirpath)
else:
path = PixivHelper.sanitizeFilename(path, rootDir)
path = path.replace('\\\\', '\\')
path = path.replace('\\', os.sep)
listItem = PixivListItem(member_id, path)
# PixivHelper.safePrint(u"- {0} ==> {1} ".format(member_id, path))
l.append(listItem)
lineNo = lineNo + 1
originalLine = ""
except UnicodeDecodeError:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'
.format(originalLine, lineNo))
except:
PixivHelper.GetLogger().exception(
"PixivListItem.parseList(): Invalid value when parsing list")
PixivHelper.printAndLog(
'error',
'Invalid value: {0} at line {1}'.format(originalLine, lineNo))
reader.close()
return l
