def init(): funcData = { "1" : {"idx" : 1, "funcName" : "UpdateNovelChapter"}, "2" : {"idx" : 2, "funcName" : "UpdateNovelLib"}, "3" : {"idx" : 3, "funcName" : "ResetNovelLib"}, "0" : {"idx" : 0, "funcName" : "Quit"}, } Helper.printLine() for data in funcData.values(): Helper.print(string = '{}: {}'.format(data.get("idx"),data.get("funcName"))) Helper.printLine() checkDone = False while not checkDone: try: data = funcData.get(str(Helper.getNum()),{}) if data: checkDone = True funcName = data.get('funcName','defalt') func = globals().get(funcName) func() except Exception as e: Helper.printError() raise e
def checkChapters(libName, libUrl, libIndex, bookName): fileContent = open(NOVEL_PATH + libName + os.sep + bookName + '.txt', 'a+', encoding='utf-8') fileMulu = open(NOVEL_PATH + libName + os.sep + bookName + '_目录.txt', 'r+', encoding='utf-8') curIdx = -1 allIdx = -1 chapterUrls = [] try: html = getBookChapterHtml(libIndex, libName, libUrl) novelName = Html.getBookName(html) if bookName == novelName: mulu = fileMulu.readlines() curIdx = len(mulu) chapterUrls = Html.getChapterUrls(html) allIdx = len(chapterUrls) else: Helper.printError("{} lib {} index {} need update".format( libName, bookName, libIndex)) except Exception as e: Helper.printError() if curIdx < allIdx: while curIdx <= (allIdx - 1): errorTimes = 0 while errorTimes < 3: try: url_name = chapterUrls[curIdx] values = re.split("=", url_name) chapter = Helper.formatChapterName(values[1]) html = getChapterHtml(libName, libUrl, libIndex, values[0]) content = Html.getChapterContent(html) fileContent.write(chapter + "\n" + content + "\n") fileMulu.write(chapter + "\n") Helper.print("{} {}".format(bookName, chapter)) errorTimes = 3 except Exception as e: errorTimes = errorTimes + 1 Helper.printError() curIdx = curIdx + 1 fileContent.close() fileMulu.close()
def checkNextUrl(): global StackFunc global StackUrls global funcIndex if StackUrls.is_empty(): Helper.print('Check WebUrl Done') else: name = StackFunc.pop() url = StackUrls.pop() if funcIndex == 1: checkBooks(name, url) elif funcIndex == 2: checkLib(name, url) else: pass
def checkLib(libName, libUrl): Helper.print("check " + libName + " lib") Novel_Lib = {} Lib = open(NOVEL_LIB_PATH + libName + '.txt', 'r+', encoding='utf-8') for line in Lib.readlines(): line = re.sub('\n', '', line) values = line.split('=') Novel_Lib[values[1]] = values[0] curIndex = len(Novel_Lib) + 1 ErrorCount = 0 while curIndex <= Lib_Max_Count: try: html = getBookChapterHtml(curIndex, libName, libUrl) novelName = Html.getBookName(html) if novelName: if Novel_Lib.get(str(curIndex), "") != novelName: ErrorCount = 0 Novel_Lib[str(curIndex)] = novelName Lib.write(novelName + "=" + str(curIndex) + "\n") Helper.print("{} add {} {}".format(libName, curIndex, novelName)) else: curIndex = curIndex + 1 else: Helper.printError(string="request {} error".format(curIndex)) ErrorCount = ErrorCount + 1 if ErrorCount >= Repeat_Max_Count: curIndex = curIndex + 1 ErrorCount = 0 except Exception as e: Helper.printError() time.sleep(Helper.randomFloat()) Lib.close() checkNextUrl()
def checkBooks(libName, libUrl): Helper.print("Check " + libName + " chapter") # 把lib里面的书按照 name=index 存到字典里 Novel_Book = {} Lib = open(NOVEL_LIB_PATH + libName + '.txt', 'r+', encoding='utf-8') for line in Lib.readlines(): line = re.sub('\n', '', line) values = line.split('=') Novel_Book[values[0]] = values[1] Lib.close() # 判断库里有没有存这本书的索引 bookSelect = 0 while bookSelect < len(SearchNovels): bookName = SearchNovels[bookSelect] Helper.print("Update {} chapters".format(bookName)) libIndex = int(Novel_Book.get(str(bookName), "0")) if libIndex != 0: checkChapters(libName, libUrl, libIndex, bookName) else: Helper.print("Can't find {} in {} lib".format(bookName, libName)) bookSelect = bookSelect + 1 time.sleep(Helper.randomFloat()) checkNextUrl()
def resetLib(): for data in URLS.values(): lib = open(NOVEL_LIB_PATH + data['name'] + '.txt', 'w') lib.close() Helper.print("reset lib done")