def parseList(filename, rootDir=None): '''read list.txt and return the list of PixivListItem''' l = list() if not os.path.exists(filename) : raise PixivException("File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION) reader = PixivHelper.OpenTextFile(filename) lineNo = 1 try: for line in reader: originalLine = line ##PixivHelper.safePrint("Processing: " + line) if line.startswith('#') or len(line) < 1: continue if len(line.strip()) == 0: continue line = PixivHelper.toUnicode(line) line = line.strip() items = line.split(" ", 1) member_id = int(items[0]) path = "" if len(items) > 1: path = items[1].strip() path = path.replace('\"', '') if rootDir != None: path = path.replace('%root%', rootDir) else: path = path.replace('%root%', '') path = os.path.abspath(path) # have drive letter if re.match(r'[a-zA-Z]:', path): dirpath = path.split(os.sep, 1) dirpath[1] = PixivHelper.sanitizeFilename(dirpath[1], None) path = os.sep.join(dirpath) else: path = PixivHelper.sanitizeFilename(path, rootDir) path = path.replace('\\\\', '\\') path = path.replace('\\', os.sep) listItem = PixivListItem(member_id, path) l.append(listItem) lineNo = lineNo + 1 originalLine = "" except UnicodeDecodeError: PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.'.format(originalLine, lineNo)) except: PixivHelper.GetLogger().exception("PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog('error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo)) reader.close() return l
def parseTagsList(filename): '''read tags.txt and return the tags list''' l = list() if not os.path.exists(filename) : raise PixivException("File doesn't exists or no permission to read: " + filename, FILE_NOT_EXISTS_OR_NO_READ_PERMISSION) reader = PixivHelper.OpenTextFile(filename) for line in reader: if line.startswith('#') or len(line) < 1: continue line = line.strip() if len(line) > 0 : l.append(PixivHelper.toUnicode(line)) reader.close() return l
def loadConfig(self, path=None): ''' new settings must be added on the last''' if path is not None: self.configFileLocation = path else: self.configFileLocation = script_path + os.sep + 'config.ini' print('Reading', self.configFileLocation, '...') haveError = False config = ConfigParser.RawConfigParser() try: config.readfp(PixivHelper.OpenTextFile(self.configFileLocation)) self.username = config.get('Authentication', 'username') self.password = config.get('Authentication', 'password') self.cookie = config.get('Authentication', 'cookie') self.tagsSeparator = PixivHelper.toUnicode(config.get('Filename', 'tagsseparator'), encoding=sys.stdin.encoding) self.rootDirectory = os.path.expanduser(PixivHelper.toUnicode(config.get('Settings', 'rootdirectory'), encoding=sys.stdin.encoding)) try: self.IrfanViewPath = os.path.expanduser( PixivHelper.toUnicode(config.get('IrfanView', 'IrfanViewPath'), encoding=sys.stdin.encoding)) self.downloadListDirectory = os.path.expanduser( PixivHelper.toUnicode(config.get('Settings', 'downloadListDirectory'), encoding=sys.stdin.encoding)) except BaseException: pass try: self.processFromDb = config.getboolean('Settings', 'processfromdb') except ValueError: print("processFromDb = True") self.processFromDb = True haveError = True try: self.dayLastUpdated = config.getint('Settings', 'daylastupdated') except ValueError: print("dayLastUpdated = 7") self.dayLastUpdated = 7 haveError = True try: self.dateDiff = config.getint('Settings', 'datediff') except ValueError: print("dateDiff = 0") self.dateDiff = 0 haveError = True try: self.proxyAddress = config.get('Network', 'proxyaddress') except ValueError: print("proxyAddress = ''") self.proxyAddress = '' haveError = True self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress} try: self.useProxy = config.getboolean('Network', 'useproxy') except ValueError: print("useProxy = False") self.useProxy = False haveError = True try: self.useList = config.getboolean('Settings', 'uselist') except ValueError: print("useList = False") self.useList = False haveError = True try: self.r18mode = config.getboolean('Pixiv', 'r18mode') except ValueError: print("r18mode = False") self.r18mode = False haveError = True _useragent = config.get('Network', 'useragent') if _useragent is not None: self.useragent = _useragent _filenameFormat = config.get('Filename', 'filenameformat') _filenameFormat = PixivHelper.toUnicode(_filenameFormat, encoding=sys.stdin.encoding) if _filenameFormat is not None and len(_filenameFormat) > 0: self.filenameFormat = _filenameFormat _filenameMangaFormat = config.get('Filename', 'filenamemangaformat') _filenameMangaFormat = PixivHelper.toUnicode(_filenameMangaFormat, encoding=sys.stdin.encoding) if _filenameMangaFormat is not None and len(_filenameMangaFormat) > 0: # check if the filename format have page identifier if not using %urlFilename% if _filenameMangaFormat.find('%urlFilename%') == -1: if _filenameMangaFormat.find('%page_index%') == -1 and _filenameMangaFormat.find('%page_number%') == -1: print('No page identifier, appending %page_index% to the filename manga format.') _filenameMangaFormat = _filenameMangaFormat + unicode(' %page_index%') print("_filenameMangaFormat =", _filenameMangaFormat) haveError = True self.filenameMangaFormat = _filenameMangaFormat _filenameInfoFormat = config.get('Filename', 'filenameinfoformat') _filenameInfoFormat = PixivHelper.toUnicode(_filenameInfoFormat, encoding=sys.stdin.encoding) if _filenameInfoFormat is not None and len(_filenameInfoFormat) > 0: self.filenameInfoFormat = _filenameInfoFormat try: self.debugHttp = config.getboolean('Debug', 'debughttp') except ValueError: self.debugHttp = False print("debugHttp = False") haveError = True try: self.useRobots = config.getboolean('Network', 'userobots') except ValueError: self.useRobots = False print("useRobots = False") haveError = True try: self.overwrite = config.getboolean('Settings', 'overwrite') except ValueError: print("overwrite = False") self.overwrite = False haveError = True try: self.createMangaDir = config.getboolean('Filename', 'createMangaDir') except ValueError: print("createMangaDir = False") self.createMangaDir = False haveError = True try: self.timeout = config.getint('Network', 'timeout') except ValueError: print("timeout = 60") self.timeout = 60 haveError = True try: self.retry = config.getint('Network', 'retry') except ValueError: print("retry = 3") self.retry = 3 haveError = True try: self.retryWait = config.getint('Network', 'retrywait') except ValueError: print("retryWait = 5") self.retryWait = 5 haveError = True try: self.numberOfPage = config.getint('Pixiv', 'numberofpage') except ValueError: self.numberOfPage = 0 print("numberOfPage = 0") haveError = True try: self.createDownloadLists = config.getboolean('IrfanView', 'createDownloadLists') except ValueError: self.createDownloadLists = False print("createDownloadLists = False") haveError = True try: self.startIrfanView = config.getboolean('IrfanView', 'startIrfanView') except ValueError: self.startIrfanView = False print("startIrfanView = False") haveError = True try: self.startIrfanSlide = config.getboolean('IrfanView', 'startIrfanSlide') except ValueError: self.startIrfanSlide = False print("startIrfanSlide = False") haveError = True try: self.alwaysCheckFileSize = config.getboolean('Settings', 'alwaysCheckFileSize') except ValueError: self.alwaysCheckFileSize = False print("alwaysCheckFileSize = False") haveError = True try: self.downloadAvatar = config.getboolean('Settings', 'downloadAvatar') except ValueError: self.downloadAvatar = False print("downloadAvatar = False") haveError = True try: self.checkUpdatedLimit = config.getint('Settings', 'checkUpdatedLimit') except ValueError: self.checkUpdatedLimit = 0 print("checkUpdatedLimit = 0") haveError = True try: self.useTagsAsDir = config.getboolean('Filename', 'useTagsAsDir') except ValueError: self.useTagsAsDir = False print("useTagsAsDir = False") haveError = True try: self.useBlacklistTags = config.getboolean('Settings', 'useBlacklistTags') except ValueError: self.useBlacklistTags = False print("useBlacklistTags = False") haveError = True try: self.useSuppressTags = config.getboolean('Settings', 'useSuppressTags') except ValueError: self.useSuppressTags = False print("useSuppressTags = False") haveError = True try: self.tagsLimit = config.getint('Settings', 'tagsLimit') except ValueError: self.tagsLimit = -1 print("tagsLimit = -1") haveError = True try: self.writeImageInfo = config.getboolean('Settings', 'writeImageInfo') except ValueError: self.writeImageInfo = False print("writeImageInfo = False") haveError = True try: self.backupOldFile = config.getboolean('Settings', 'backupOldFile') except ValueError: self.backupOldFile = False print("backupOldFile = False") haveError = True try: self.logLevel = config.get('Debug', 'logLevel').upper() if self.logLevel not in ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']: raise ValueError("Value not in list: " + self.logLevel) except ValueError: print("logLevel = DEBUG") self.logLevel = 'DEBUG' haveError = True try: self.enableDump = config.getboolean('Debug', 'enableDump') except ValueError: print("enableDump = True") self.enableDump = True haveError = True try: self.skipDumpFilter = config.get('Debug', 'skipDumpFilter') except ValueError: print("skipDumpFilter = ''") self.skipDumpFilter = '' haveError = True try: self.dumpMediumPage = config.getboolean('Debug', 'dumpMediumPage') except ValueError: print("dumpMediumPage = False") self.dumpMediumPage = False haveError = True try: self.enableInfiniteLoop = config.getboolean('Settings', 'enableInfiniteLoop') except ValueError: self.enableInfiniteLoop = False print("enableInfiniteLoop = False") haveError = True try: self.dumpTagSearchPage = config.getboolean('Debug', 'dumpTagSearchPage') except ValueError: self.dumpTagSearchPage = False print("dumpTagSearchPage = False") haveError = True try: self.dateFormat = config.get('Pixiv', 'dateFormat') except ValueError: print("dateFormat = ''") self.dateFormat = '' haveError = True try: self.verifyImage = config.getboolean('Settings', 'verifyImage') except ValueError: print("verifyImage = False") self.verifyImage = False haveError = True try: self.writeUrlInDescription = config.getboolean('Settings', 'writeUrlInDescription') except ValueError: print("writeUrlInDescription = False") self.writeUrlInDescription = False haveError = True try: self.urlBlacklistRegex = config.get('Settings', 'urlBlacklistRegex') except ValueError: print("urlBlacklistRegex = ") self.urlBlacklistRegex = "" haveError = True try: self.urlDumpFilename = config.get('Filename', 'urlDumpFilename') except ValueError: print("urlDumpFilename = url_list_%Y%m%d") self.urlDumpFilename = "url_list_%Y%m%d" haveError = True try: self.dbPath = config.get('Settings', 'dbPath') except ValueError: print("dbPath = ''") self.dbPath = '' haveError = True try: self.useBlacklistMembers = config.getboolean('Settings', 'useBlacklistMembers') except ValueError: print("useBlacklistMembers = False") self.useBlacklistMembers = False haveError = True try: self.avatarNameFormat = config.get('Filename', 'avatarNameFormat') self.avatarNameFormat = PixivHelper.toUnicode(self.avatarNameFormat, encoding=sys.stdin.encoding) except ValueError: print("avatarNameFormat = ") self.avatarNameFormat = "" haveError = True try: self.writeImageJSON = config.getboolean('Settings', 'writeImageJSON') except ValueError: self.writeImageJSON = False print("writeImageJSON = False") haveError = True try: self.downloadDelay = config.getint('Network', 'downloadDelay') except ValueError: self.downloadDelay = 2 print("downloadDelay = 2") haveError = True try: self.ffmpeg = config.get('FFmpeg', 'ffmpeg') except ValueError: print("ffmpeg = 'ffmpeg'") self.ffmpeg = 'ffmpeg' haveError = True try: self.ffmpegCodec = config.get('FFmpeg', 'ffmpegCodec') except ValueError: print("ffmpegCodec = 'libvpx-vp9'") self.ffmpegCodec = 'libvpx-vp9' haveError = True try: self.ffmpegParam = config.get('FFmpeg', 'ffmpegParam') except ValueError: print("ffmpegParam = '-lossless 1'") self.ffmpegParam = '-lossless 1' haveError = True try: self.setLastModified = config.getboolean('Settings', 'setLastModified') except ValueError: print("setLastModified = True") self.setLastModified = True haveError = True try: self.useLocalTimezone = config.getboolean('Settings', 'useLocalTimezone') except ValueError: print("useLocalTimezone = False") self.useLocalTimezone = False haveError = True try: self.checkNewVersion = config.getboolean('Network', 'checkNewVersion') except ValueError: print("checkNewVersion = True") self.checkNewVersion = True haveError = True try: self.webpCodec = config.get('FFmpeg', 'webpCodec') except ValueError: print("webpCodec = 'libwebp'") self.webpCodec = 'libwebp' haveError = True try: self.webpParam = config.get('FFmpeg', 'webpParam') except ValueError: print("webpParam = '-lossless 0 -q:v 90'") self.webpParam = '-lossless 0 -q:v 90' haveError = True try: self.createWebp = config.getboolean('Ugoira', 'createWebp') except ValueError: print("createWebp = False") self.createWebp = False haveError = True try: self.writeUgoiraInfo = config.getboolean('Ugoira', 'writeUgoiraInfo') except ValueError: self.writeUgoiraInfo = False print("writeUgoiraInfo = False") haveError = True try: self.createUgoira = config.getboolean('Ugoira', 'createUgoira') except ValueError: self.createUgoira = False print("createUgoira = False") haveError = True try: self.deleteZipFile = config.getboolean('Ugoira', 'deleteZipFile') except ValueError: self.deleteZipFile = False print("deleteZipFile = False") haveError = True try: self.createGif = config.getboolean('Ugoira', 'createGif') except ValueError: print("createGif = False") self.createGif = False haveError = True try: self.createApng = config.getboolean('Ugoira', 'createApng') except ValueError: print("createApng = False") self.createApng = False haveError = True try: self.deleteUgoira = config.getboolean('Ugoira', 'deleteUgoira') except ValueError: print("deleteUgoira = False") self.deleteUgoira = False haveError = True try: self.createWebm = config.getboolean('Ugoira', 'createWebm') except ValueError: print("createWebm = False") self.createWebm = False haveError = True try: self.refresh_token = config.get('Authentication', 'refresh_token') except ValueError: print("refresh_token = ''") self.refresh_token = None haveError = True try: self.enableSSLVerification = config.getboolean('Network', 'enableSSLVerification') except ValueError: print("enableSSLVerification = False") self.enableSSLVerification = False haveError = True except BaseException: print('Error at loadConfig():', sys.exc_info()) self.__logger.exception('Error at loadConfig()') haveError = True if haveError: print('Some configuration have invalid value, replacing with the default value.') self.writeConfig(error=True) print('done.')
def loadConfig(self, path=None): ''' new settings must be added on the last''' if path != None: self.configFileLocation = path else: self.configFileLocation = script_path + os.sep + 'config.ini' print 'Reading', self.configFileLocation, '...' haveError = False config = ConfigParser.RawConfigParser() try: config.readfp(PixivHelper.OpenTextFile(self.configFileLocation)) self.username = config.get('Authentication','username') self.password = config.get('Authentication','password') self.cookie = config.get('Authentication','cookie') self.tagsSeparator = PixivHelper.toUnicode(config.get('Settings','tagsseparator'), encoding=sys.stdin.encoding) self.rootDirectory = os.path.expanduser(PixivHelper.toUnicode(config.get('Settings','rootdirectory'), encoding=sys.stdin.encoding)) try: self.IrfanViewPath = os.path.expanduser( PixivHelper.toUnicode(config.get('IrfanView','IrfanViewPath'), encoding=sys.stdin.encoding)) self.downloadListDirectory = os.path.expanduser( PixivHelper.toUnicode(config.get('Settings','downloadListDirectory'), encoding=sys.stdin.encoding)) except: pass try: self.processFromDb = config.getboolean('Settings','processfromdb') except ValueError: print "processFromDb = True" self.processFromDb = True haveError = True try: self.dayLastUpdated = config.getint('Settings','daylastupdated') except ValueError: print "dayLastUpdated = 7" self.dayLastUpdated = 7 haveError = True try: self.dateDiff = config.getint('Settings','datediff') except ValueError: print "dateDiff = 0" self.dateDiff = 0 haveError = True try: self.proxyAddress = config.get('Network','proxyaddress') except ValueError: print "proxyAddress = ''" self.proxyAddress = '' haveError = True self.proxy = {'http': self.proxyAddress, 'https': self.proxyAddress} try: self.useProxy = config.getboolean('Network','useproxy') except ValueError: print "useProxy = False" self.useProxy = False haveError = True try: self.useList = config.getboolean('Settings','uselist') except ValueError: print "useList = False" self.useList = False haveError = True try: self.r18mode = config.getboolean('Pixiv','r18mode') except ValueError: print "r18mode = False" self.r18mode = False haveError = True _useragent = config.get('Network','useragent') if _useragent != None: self.useragent = _useragent _filenameFormat = config.get('Settings','filenameformat') _filenameFormat = PixivHelper.toUnicode(_filenameFormat, encoding=sys.stdin.encoding) if _filenameFormat != None: self.filenameFormat = _filenameFormat _filenameMangaFormat = config.get('Settings','filenamemangaformat') _filenameMangaFormat = PixivHelper.toUnicode(_filenameMangaFormat, encoding=sys.stdin.encoding) if _filenameMangaFormat != None: ## check if the filename format have page identifier if not using %urlFilename% if _filenameMangaFormat.find('%urlFilename%') == -1: if _filenameMangaFormat.find('%page_index%') == -1 and _filenameMangaFormat.find('%page_number%') == -1: print 'No page identifier, appending %page_index% to the filename manga format.' _filenameMangaFormat = _filenameMangaFormat + unicode(' %page_index%') print "_filenameMangaFormat =", _filenameMangaFormat haveError = True self.filenameMangaFormat = _filenameMangaFormat try: self.debugHttp = config.getboolean('Debug','debughttp') except ValueError: self.debugHttp = False print "debugHttp = False" haveError = True try: self.useRobots = config.getboolean('Network','userobots') except ValueError: self.useRobots = False print "useRobots = False" haveError = True try: self.overwrite = config.getboolean('Settings','overwrite') except ValueError: print "overwrite = False" self.overwrite = False haveError = True try: self.createMangaDir = config.getboolean('Settings','createMangaDir') except ValueError: print "createMangaDir = False" self.createMangaDir = False haveError = True try: self.timeout = config.getint('Network','timeout') except ValueError: print "timeout = 60" self.timeout = 60 haveError = True try: self.retry = config.getint('Network','retry') except ValueError: print "retry = 3" self.retry = 3 haveError = True try: self.retryWait = config.getint('Network','retrywait') except ValueError: print "retryWait = 5" self.retryWait = 5 haveError = True try: self.numberOfPage = config.getint('Pixiv','numberofpage') except ValueError: self.numberOfPage = 0 print "numberOfPage = 0" haveError = True try: self.createDownloadLists = config.getboolean('IrfanView','createDownloadLists') except ValueError: self.createDownloadLists = False print "createDownloadLists = False" haveError = True try: self.startIrfanView = config.getboolean('IrfanView','startIrfanView') except ValueError: self.startIrfanView = False print "startIrfanView = False" haveError = True try: self.startIrfanSlide = config.getboolean('IrfanView','startIrfanSlide') except ValueError: self.startIrfanSlide = False print "startIrfanSlide = False" haveError = True try: self.alwaysCheckFileSize = config.getboolean('Settings','alwaysCheckFileSize') except ValueError: self.alwaysCheckFileSize = False print "alwaysCheckFileSize = False" haveError = True try: self.downloadAvatar = config.getboolean('Settings','downloadAvatar') except ValueError: self.downloadAvatar = False print "downloadAvatar = False" haveError = True try: self.checkUpdatedLimit = config.getint('Settings','checkUpdatedLimit') except ValueError: self.checkUpdatedLimit = 0 print "checkUpdatedLimit = 0" haveError = True try: self.useTagsAsDir = config.getboolean('Settings','useTagsAsDir') except ValueError: self.useTagsAsDir = False print "useTagsAsDir = False" haveError = True try: self.useBlacklistTags = config.getboolean('Settings','useBlacklistTags') except ValueError: self.useBlacklistTags = False print "useBlacklistTags = False" haveError = True try: self.useSuppressTags = config.getboolean('Settings','useSuppressTags') except ValueError: self.useSuppressTags = False print "useSuppressTags = False" haveError = True try: self.tagsLimit = config.getint('Settings','tagsLimit') except ValueError: self.tagsLimit = -1 print "tagsLimit = -1" haveError = True try: self.useSSL = config.getboolean('Authentication','useSSL') except ValueError: self.useSSL = False print "useSSL = False" haveError = True try: self.writeImageInfo = config.getboolean('Settings','writeImageInfo') except ValueError: self.writeImageInfo = False print "writeImageInfo = False" haveError = True try: self.keepSignedIn = config.getint('Authentication','keepSignedIn') except ValueError: print "keepSignedIn = 0" self.keepSignedIn = 0 haveError = True try: self.backupOldFile = config.getboolean('Settings','backupOldFile') except ValueError: self.backupOldFile = False print "backupOldFile = False" haveError = True try: self.logLevel = config.get('Debug','logLevel').upper() if not self.logLevel in ['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO', 'DEBUG', 'NOTSET']: raise ValueError("Value not in list: " + self.logLevel) except ValueError: print "logLevel = DEBUG" self.logLevel = 'DEBUG' haveError = True try: self.enableDump = config.getboolean('Debug','enableDump') except ValueError: print "enableDump = True" self.enableDump = True haveError = True try: self.skipDumpFilter = config.get('Debug','skipDumpFilter') except ValueError: print "skipDumpFilter = ''" self.skipDumpFilter = '' haveError = True try: self.dumpMediumPage = config.getboolean('Debug','dumpMediumPage') except ValueError: print "dumpMediumPage = False" self.dumpMediumPage = False haveError = True try: self.writeUgoiraInfo = config.getboolean('Settings','writeUgoiraInfo') except ValueError: self.writeUgoiraInfo = False print "writeUgoiraInfo = False" haveError = True try: self.createUgoira = config.getboolean('Settings','createUgoira') except ValueError: self.createUgoira = False print "createUgoira = False" haveError = True try: self.deleteZipFile = config.getboolean('Settings','deleteZipFile') except ValueError: self.deleteZipFile = False print "deleteZipFile = False" haveError = True try: self.enableInfiniteLoop = config.getboolean('Settings','enableInfiniteLoop') except ValueError: self.enableInfiniteLoop = False print "enableInfiniteLoop = False" haveError = True try: self.dumpTagSearchPage = config.getboolean('Debug','dumpTagSearchPage') except ValueError: self.dumpTagSearchPage = False print "dumpTagSearchPage = False" haveError = True try: self.dateFormat = config.get('Pixiv','dateFormat') except ValueError: print "dateFormat = ''" self.dateFormat = '' haveError = True try: self.verifyImage = config.getboolean('Settings','verifyImage') except ValueError: print "verifyImage = False" self.verifyImage = False haveError = True try: self.writeUrlInDescription = config.getboolean('Settings','writeUrlInDescription') except ValueError: print "writeUrlInDescription = False" self.writeUrlInDescription = False haveError = True ## except ConfigParser.NoOptionError: ## print 'Error at loadConfig():',sys.exc_info() ## print 'Failed to read configuration.' ## self.writeConfig() ## except ConfigParser.NoSectionError: ## print 'Error at loadConfig():',sys.exc_info() ## print 'Failed to read configuration.' ## self.writeConfig() except: print 'Error at loadConfig():',sys.exc_info() self.__logger.exception('Error at loadConfig()') haveError = True if haveError: print 'Some configuration have invalid value, replacing with the default value.' self.writeConfig(error=True) print 'done.'
def parseList(filename, rootDir=None): '''read list.txt and return the list of PixivListItem''' l = list() if not os.path.exists(filename): raise PixivException( "File doesn't exists or no permission to read: " + filename, errorCode=PixivException.FILE_NOT_EXISTS_OR_NO_WRITE_PERMISSION ) reader = PixivHelper.OpenTextFile(filename) lineNo = 1 try: for line in reader: originalLine = line # PixivHelper.safePrint("Processing: " + line) if line.startswith('#') or len(line) < 1: continue if len(line.strip()) == 0: continue line = PixivHelper.toUnicode(line) line = line.strip() items = line.split(None, 1) if items[0].startswith("http"): # handle urls: # http://www.pixiv.net/member_illust.php?id=<member_id> # http://www.pixiv.net/member.php?id=<member_id> parsed = urlparse.urlparse(items[0]) if parsed.path == "/member.php" or parsed.path == "/member_illust.php": query_str = urlparse.parse_qs(parsed.query) if query_str.has_key("id"): member_id = int(query_str["id"][0]) else: PixivHelper.printAndLog( 'error', "Cannot detect member id from url: " + items[0]) continue else: PixivHelper.printAndLog( 'error', "Unsupported url detected: " + items[0]) continue else: # handle member id directly member_id = int(items[0]) path = "" if len(items) > 1: path = items[1].strip() path = path.replace('\"', '') if rootDir is not None: path = path.replace('%root%', rootDir) else: path = path.replace('%root%', '') path = os.path.abspath(path) # have drive letter if re.match(r'[a-zA-Z]:', path): dirpath = path.split(os.sep, 1) dirpath[1] = PixivHelper.sanitizeFilename( dirpath[1], None) path = os.sep.join(dirpath) else: path = PixivHelper.sanitizeFilename(path, rootDir) path = path.replace('\\\\', '\\') path = path.replace('\\', os.sep) listItem = PixivListItem(member_id, path) # PixivHelper.safePrint(u"- {0} ==> {1} ".format(member_id, path)) l.append(listItem) lineNo = lineNo + 1 originalLine = "" except UnicodeDecodeError: PixivHelper.GetLogger().exception( "PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog( 'error', 'Invalid value: {0} at line {1}, try to save the list.txt in UTF-8.' .format(originalLine, lineNo)) except: PixivHelper.GetLogger().exception( "PixivListItem.parseList(): Invalid value when parsing list") PixivHelper.printAndLog( 'error', 'Invalid value: {0} at line {1}'.format(originalLine, lineNo)) reader.close() return l