import Dictionary import FSBuilder import AccEval import glob DataDirectory = "data/" debug = True FSCheck = True if debug: print("Verbose feedback enabled\n") Lexicon = Dictionary.BuildLexicon(debug) if FSCheck: FSSub = FSBuilder.BuildFS(Lexicon,debug) else: FSSub = set() report = list() if debug: print("Building file list from " + DataDirectory) TextList = glob.glob(DataDirectory + "*.txt") print("Preparing score report for " + str(len(TextList) ) + " items in " + DataDirectory + "\n")
else: startindex = 0 HTIDfile = metadatapath + "htids.txt" with open(HTIDfile, encoding="utf-8") as file: HTIDlist = file.readlines() if startindex >= len(HTIDlist): print("Finished processing the whole list of volume IDs.") quit() elif startindex + batchcount > len(HTIDlist): endindex = len(HTIDlist) else: endindex = startindex + batchcount Lexicon = Dictionary.BuildLexicon(dictionarypath, debug) writename = 'typeindex.txt' delim = '\t' if startindex == 0: BigIndex = dict() else: BigIndex = TypeIndex.ReadIndex(writename, delim, debug) SortedIndex = list() for index in range(startindex, endindex): IDtoprocess = HTIDlist[index].strip() filepath, postfix = FileCabinet.pairtreepath(IDtoprocess, datapath)
import Dictionary import ReadText import hyphenread1 import hyphenread2 import FSBuilder Lexicon = Dictionary.BuildLexicon(True) FSSub = FSBuilder.BuildFS(Lexicon, True) ## Text = ReadText.CleanText("AustBCR.txt", True) Text = hyphenread1.CleanText("AustBCR.txt", Lexicon, True) Match = 0 Sub = 0 Miss = set() for word in Text: if word in Lexicon: Match = Match + 1 continue elif word in FSSub: Match = Match + 1 Sub = Sub + 1 continue print("Base Matches: " + str(Match - Sub)) print("Sub. Matches: " + str(Sub)) print("Total Matches: " + str(Match) + "\n")