def process_a_file(file_tuple): global testrun, pairtreepath, datapath, genremapdir, felecterrors, selecttruths, debug, phraseset, pagevocabset, meaningfulheaders thisID, metadata_evidence = file_tuple perfileerrorlog = list() return_dict = dict() return_dict["htid"] = thisID return_dict["metadata"] = (thisID, "0", "0", "0", "0", "0") return_dict["errors"] = [] return_dict["phrasecounts"] = dict() if testrun: cleanID = clean_pairtree(thisID.replace("norm.txt", "")) else: cleanID = clean_pairtree(thisID) if not testrun: filepath, postfix = FileCabinet.pairtreepath(thisID, datapath) filename = filepath + postfix + '/' + postfix + ".zip" else: filename = datapath + thisID # ACTUALLY READ THE FILE. if filename.endswith('.zip'): pagelist, successflag = read_zip(filename) else: pagelist, successflag = read_txt(filename) if successflag == "missing file": print(thisID + " is missing.") perfileerrorlog.append(thisID + '\t' + "missing") return_dict["errors"] = perfileerrorlog return return_dict elif successflag == "pagination error": print(thisID + " has a pagination problem.") perfileerrorlog.append(thisID + '\t' + "paginationerror") return_dict["errors"] = perfileerrorlog return return_dict elif successflag == "unicode error": print(thisID + " can not be decoded by unicode.") perfileerrorlog.append(thisID + '\t' + "unicode error") return_dict["errors"] = perfileerrorlog return return_dict tokens, pre_matched, pre_english, pagedata, headerlist = NormalizeVolume.as_stream(pagelist, verbose=debug) if pre_english < 0.6: perfileerrorlog.append(thisID + '\t' + "not english") tokencount = len(tokens) if len(tokens) < 10: print(thisID, "has only tokencount", len(tokens)) perfileerrorlog.append(thisID + '\t' + 'short') correct_tokens, pages, post_matched, post_english = NormalizeVolume.correct_stream(tokens, verbose = debug) # Combine page dictionaries into a master dictionary. # If you ask, why didn't you just produce one in the first place? ... # answer has to do with flexibility of the Volume module for other purposes. pagecounter = 0 masterdict = dict() for page in pages: for item in page: if item in masterdict: masterdict[item] += page[item] else: masterdict[item] = page[item] # Now that we have a master dictionary, consider whether there are long-s problems. # This algorithm works adequately. errors = 1 truths = 1 # Initialized to 1 as a Laplacian correction. for word in felecterrors: errors = errors + masterdict.get(word, 0) for word in selecttruths: truths = truths + masterdict.get(word, 0) if truths > errors: LongSproblem = False else: LongSproblem = True if LongSproblem == False: corrected = correct_tokens deleted = dict() added = dict() else: deleted, added, corrected, changedphrases, unchanged = Context.catch_ambiguities(correct_tokens, debug) # okay, this is crazy and not efficient to run, but it's easy to write and there are a small number # of these files -- so I'm going to count the new contextually-corrected tokens by re-running them # through Volume. correct_tokens, pages, post_matched, post_english = NormalizeVolume.correct_stream(corrected, verbose = debug) corrected = correct_tokens # If we are upvoting tokens in the header, they need to be added here. if len(pages) != len(headerlist): print(thisID + " fails a routine check of alignment between pages and headers.") else: for index, page in enumerate(pages): thispageheader = headerlist[index] header_tokens, header_pages, dummy1, dummy2 = NormalizeVolume.correct_stream(thispageheader, verbose = debug) headerdict = header_pages[0] for key, value in headerdict.items(): if key in meaningfulheaders: if key in page: page[key] += 2 # a fixed increment no matter how many times the word occurs in the # header else: page[key] = 2 print("Word " + key + " in headerdict for " + thisID + " at " + str(index) + " but not main page.") # Write corrected file. cleanHTID = clean_pairtree(thisID) if testrun: if cleanHTID.endswith(".clean.txt"): outHTID = cleanHTID.replace(".clean.txt", "") elif cleanHTID.endswith("norm.txt"): outHTID = cleanHTID.replace("norm.txt", ".norm.txt") elif cleanHTID.endswith(".txt"): outHTID = cleanHTID.replace(".txt", "norm.txt") else: outHTID = cleanHTID + ".norm.txt" outfilename = outpath + "texts/" + outHTID else: outfilename = filepath + postfix + '/' + postfix + ".norm.txt" with open(outfilename, mode = 'w', encoding = 'utf-8') as file: for token in corrected: if token != '\n' and token != "“" and not (token.startswith('<') and token.endswith('>')): token = token + " " file.write(token) if len(pages) != len(pagedata): perfileerrorlog.append("Discrepancy between page data and page metadata in \t" + thisID) return_dict["errors"] = perfileerrorlog return return_dict totalwordsinvol = 0 if testrun: if cleanHTID.endswith(".clean.txt"): outHTID = cleanHTID.replace(".clean.txt", ".pg.tsv") elif cleanHTID.endswith("norm.txt"): outHTID = cleanHTID.replace("norm.txt", ".pg.tsv") elif cleanHTID.endswith(".txt"): outHTID = cleanHTID.replace(".txt", ".pg.tsv") else: outHTID = cleanHTID + ".pg.tsv" outfilename = outpath + "pagefeatures/" + outHTID else: outfilename = filepath + postfix + '/' + postfix + ".pg.tsv" with open(outfilename, mode = 'w', encoding = 'utf-8') as file: if metadata_evidence["biography"]: file.write("-1\t#metaBiography\t0\n") if metadata_evidence["drama"]: file.write("-1\t#metaDrama\t0\n") if metadata_evidence["fiction"]: file.write("-1\t#metaFiction\t0\n") if metadata_evidence["poetry"]: file.write("-1\t#metaPoetry\t0\n") numberofpages = len(pages) for index, page in enumerate(pages): # This is a shameful hack that should be deleted later. if testrun and "estimated" in page and "percentage" in page and (index + 3) > numberofpages: continue if testrun and "untypical" in page and (index +2) > numberofpages: continue otherfeatures = 0 for feature, count in page.items(): if feature in pagevocabset or feature.startswith("#"): outline = str(index) + '\t' + feature + '\t' + str(count) + '\n' # pagenumber, featurename, featurecount file.write(outline) else: otherfeatures += count if not feature.startswith("#"): totalwordsinvol += count # This is because there are structural features like #allcapswords # that should not be counted toward total token count. structural_features = pagedata[index] for feature, count in structural_features.items(): if count > 0 or feature == "#textlines": outline = str(index) + '\t' + feature + '\t' + str(count) + '\n' file.write(outline) if otherfeatures > 0: outline = str(index) + '\t' + "wordNotInVocab" + '\t' + str(otherfeatures) + '\n' file.write(outline) metatuple = (thisID, str(totalwordsinvol), str(pre_matched), str(pre_english), str(post_matched), str(post_english)) return_dict["metadata"] = metatuple return_dict["errors"] = perfileerrorlog return return_dict
def process_a_file(file_tuple): global testrun, pairtreepath, datapath, genremapdir, felecterrors, selecttruths, debug, phraseset, pagevocabset, meaningfulheaders thisID, metadata_evidence = file_tuple perfileerrorlog = list() return_dict = dict() return_dict["htid"] = thisID return_dict["metadata"] = (thisID, "0", "0", "0", "0", "0") return_dict["errors"] = [] return_dict["phrasecounts"] = dict() if testrun: cleanID = clean_pairtree(thisID.replace("norm.txt", "")) else: cleanID = clean_pairtree(thisID) if not testrun: filepath, postfix = FileCabinet.pairtreepath(thisID, datapath) filename = filepath + postfix + '/' + postfix + ".zip" else: filename = datapath + thisID # ACTUALLY READ THE FILE. if filename.endswith('.zip'): pagelist, successflag = read_zip(filename) else: pagelist, successflag = read_txt(filename) if successflag == "missing file": print(thisID + " is missing.") perfileerrorlog.append(thisID + '\t' + "missing") return_dict["errors"] = perfileerrorlog return return_dict elif successflag == "pagination error": print(thisID + " has a pagination problem.") perfileerrorlog.append(thisID + '\t' + "paginationerror") return_dict["errors"] = perfileerrorlog return return_dict tokens, pre_matched, pre_english, pagedata, headerlist = NormalizeVolume.as_stream(pagelist, verbose=debug) if pre_english < 0.6: perfileerrorlog.append(thisID + '\t' + "not english") tokencount = len(tokens) if len(tokens) < 10: print(thisID, "has only tokencount", len(tokens)) perfileerrorlog.append(thisID + '\t' + 'short') correct_tokens, pages, post_matched, post_english = NormalizeVolume.correct_stream(tokens, verbose = debug) # Combine page dictionaries into a master dictionary. # If you ask, why didn't you just produce one in the first place? ... # answer has to do with flexibility of the Volume module for other purposes. pagecounter = 0 masterdict = dict() for page in pages: for item in page: if item in masterdict: masterdict[item] += page[item] else: masterdict[item] = page[item] # Now that we have a master dictionary, consider whether there are long-s problems. # This algorithm works adequately. errors = 1 truths = 1 # Initialized to 1 as a Laplacian correction. for word in felecterrors: errors = errors + masterdict.get(word, 0) for word in selecttruths: truths = truths + masterdict.get(word, 0) if truths > errors: LongSproblem = False else: LongSproblem = True if LongSproblem == False: corrected = correct_tokens deleted = dict() added = dict() else: deleted, added, corrected, changedphrases, unchanged = Context.catch_ambiguities(correct_tokens, debug) # okay, this is crazy and not efficient to run, but it's easy to write and there are a small number # of these files -- so I'm going to count the new contextually-corrected tokens by re-running them # through Volume. correct_tokens, pages, post_matched, post_english = NormalizeVolume.correct_stream(corrected, verbose = debug) corrected = correct_tokens # If we are upvoting tokens in the header, they need to be added here. for index, page in enumerate(pages): thispageheader = headerlist[index] header_tokens, header_pages, dummy1, dummy2 = NormalizeVolume.correct_stream(thispageheader, verbose = debug) headerdict = header_pages[0] for key, value in headerdict.items(): if key in meaningfulheaders: if key in page: page[key] += 2 # a fixed increment no matter how many times the word occurs in the # header else: page[key] = 2 print("Word " + key + " in headerdict for " + thisID + " at " + str(index) + " but not main page.") # Write corrected file. cleanHTID = clean_pairtree(thisID) if testrun: if cleanHTID.endswith(".clean.txt"): outHTID = cleanHTID.replace(".clean.txt", "") elif cleanHTID.endswith("norm.txt"): outHTID = cleanHTID.replace("norm.txt", ".norm.txt") elif cleanHTID.endswith(".txt"): outHTID = cleanHTID.replace(".txt", "norm.txt") else: outHTID = cleanHTID + ".norm.txt" outfilename = outpath + "texts/" + outHTID else: outfilename = filepath + postfix + '/' + postfix + ".norm.txt" with open(outfilename, mode = 'w', encoding = 'utf-8') as file: for token in corrected: if token != '\n' and token != "“" and not (token.startswith('<') and token.endswith('>')): token = token + " " file.write(token) if len(pages) != len(pagedata): perfileerrorlog.append("Discrepancy between page data and page metadata in \t" + thisID) totalwordsinvol = 0 if testrun: if cleanHTID.endswith(".clean.txt"): outHTID = cleanHTID.replace(".clean.txt", ".pg.tsv") elif cleanHTID.endswith("norm.txt"): outHTID = cleanHTID.replace("norm.txt", ".pg.tsv") elif cleanHTID.endswith(".txt"): outHTID = cleanHTID.replace(".txt", ".pg.tsv") else: outHTID = cleanHTID + ".pg.tsv" outfilename = outpath + "pagefeatures/" + outHTID else: outfilename = filepath + postfix + '/' + postfix + ".pg.tsv" with open(outfilename, mode = 'w', encoding = 'utf-8') as file: if metadata_evidence["biography"]: file.write("-1\t#metaBiography\t0\n") if metadata_evidence["drama"]: file.write("-1\t#metaDrama\t0\n") if metadata_evidence["fiction"]: file.write("-1\t#metaFiction\t0\n") if metadata_evidence["poetry"]: file.write("-1\t#metaPoetry\t0\n") numberofpages = len(pages) for index, page in enumerate(pages): # This is a shameful hack that should be deleted later. if testrun and "estimated" in page and "percentage" in page and (index + 3) > numberofpages: continue if testrun and "untypical" in page and (index +2) > numberofpages: continue otherfeatures = 0 for feature, count in page.items(): if feature in pagevocabset or feature.startswith("#"): outline = str(index) + '\t' + feature + '\t' + str(count) + '\n' # pagenumber, featurename, featurecount file.write(outline) else: otherfeatures += count if not feature.startswith("#"): totalwordsinvol += count # This is because there are structural features like #allcapswords # that should not be counted toward total token count. structural_features = pagedata[index] for feature, count in structural_features.items(): if count > 0 or feature == "#textlines": outline = str(index) + '\t' + feature + '\t' + str(count) + '\n' file.write(outline) if otherfeatures > 0: outline = str(index) + '\t' + "wordNotInVocab" + '\t' + str(otherfeatures) + '\n' file.write(outline) metatuple = (thisID, str(totalwordsinvol), str(pre_matched), str(pre_english), str(post_matched), str(post_english)) return_dict["metadata"] = metatuple return_dict["errors"] = perfileerrorlog return return_dict
Lexicon = Dictionary.BuildLexicon(dictionarypath, debug) writename = 'typeindex.txt' delim = '\t' if startindex == 0: BigIndex = dict() else: BigIndex = TypeIndex.ReadIndex(writename, delim, debug) SortedIndex = list() for index in range(startindex, endindex): IDtoprocess = HTIDlist[index].strip() filepath, postfix = FileCabinet.pairtreepath(IDtoprocess, datapath) filename = filepath + postfix + '/' + postfix + ".txt" try: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(IDtoprocess + " is missing.") continue tokens = TokenGen.keep_hyphens(lines, Lexicon, verbose=debug)
def main(): import FileCabinet import FileUtils import Volume2 import Context import sys import os # DEFINE CONSTANTS. delim = '\t' debug = False felecterrors = [ 'fee', 'fea', 'fay', 'fays', 'fame', 'fell', 'funk', 'fold', 'haft', 'fat', 'fix', 'chafe', 'loft' ] selecttruths = [ 'see', 'sea', 'say', 'says', 'same', 'sell', 'sunk', 'sold', 'hast', 'sat', 'six', 'chase', 'lost' ] # Locate ourselves in the directory structure. cwd = os.getcwd() cwdparent = os.path.abspath(os.path.join(cwd, os.pardir)) # We need to find a directory called 'rulesets,' which we expect to be located # either within the working directory or adjacent to it. if os.path.isdir(os.path.join(cwd, "rulesets")): rulepath = os.path.join(cwd, "rulesets") elif os.path.isdir(os.path.join(cwdparent, "rulesets")): rulepath = os.path.join(cwdparent, "rulesets") else: user = input("Please specify a path to the ruleset directory: ") if os.path.isdir(user): rulepath = user else: print("Invalid path.") sys.exit() # Use rulepath to load relevant rules inside modules. Volume2.importrules(rulepath) Context.importrules(rulepath) # Now we enter dialogue with the user. This is all a little 1982, # but what can I say? Wetware upgrades are expensive. def prompt(promptstring, options): user = input(promptstring) if user not in options: user = prompt(promptstring, options) return user # Ask the user to tell us how to find files to process. print("****************** CorrectOCR 0.1 ******************") print() print("Do you want the full spiel (explanations, caveats, etc.)") user = prompt("y/n : ", ["y", "n"]) if user.lower() == "y": spielpath = os.path.join(cwd, "spiel.txt") with open(spielpath, encoding='utf-8') as file: filelines = file.readlines() for line in filelines: print(line, end='') print("\nThis script will correct .txt files, or extract text") print("from zipped archives containing one txt file for each page.") print("In either case it writes the cleaned files back to their") print("original locations with the new suffix '.clean.txt'.") print("\nDo you want to unpack .zip files or .txt files?") user = prompt("zip or txt: ", ["zip", "txt"]) suffix = "." + user suffixlen = len(suffix) print("\nThere are two ways to identify the location of the") print("files to be corrected.") print("\n1. Provide the path to a folder that contains them. I'll") print("recursively search subdirectories of that folder as well. Or,") print("\n2. Provide a file holding a list of pairtree file identifiers,") print("e.g. HathiTrust Volume IDs. I can use those identifiers to infer") print("the paths to the files themselves.\n") user = prompt("Which option do you prefer (1 or 2)? ", ["1", "2"]) if user == "1": rootpath = input("Path to the folder that contains source files: ") filelist = FileUtils.recursivefilegetter(rootpath, suffix) else: print("I expect the pairtree identifiers to be listed one per line,") print("and to be the only item on a line.") filepath = input( "Path to the file that contains pairtree identifiers: ") filelist = list() with open(filepath, encoding='utf-8') as file: filelines = file.readlines() print( "Now I need a path to the folder that contains the pairtree structure." ) print( "If you have multiple folders for different libraries, this should be" ) print("the folder above them all. It should end with a slash.") rootpath = input("Path to the folder that contains pairtree: ") for line in filelines: line = line.rstrip() filepath, postfix = FileCabinet.pairtreepath(line, rootpath) filename = filepath + postfix + '/' + postfix + suffix filelist.append(filename) print("\nI identified", len(filelist), "files in that location.") print("\nI can just write clean text files (with suffix clean.txt)") print("or I can also write tab-separated files that count the words") print("in each file after correction.") user = prompt("1) Text only or 2) text-plus-wordcounts? (1 or 2): ", ["1", "2"]) if user == "1": wordcountflag = False else: wordcountflag = True print("Now proceeding to process the files.\n") def subtract_counts(token, adict, tosubtract): '''Adjusts a dictionary by subtracting tosubtract instances of token.''' if token in adict: adict[token] = adict[token] - tosubtract if adict[token] < 0: del adict[token] elif adict[token] < 1: del adict[token] return adict def add_counts(token, adict, toadd): '''Adjusts a dictionary by adding toadd instances of token.''' if token in adict: adict[token] = adict[token] + toadd else: adict[token] = toadd return adict # Here's where we BEGIN THE ACTUAL CORRECTION OF FILES. processedmeta = list() errorlog = list() longSfiles = list() count = 0 for filename in filelist: try: if suffix == ".zip": lines = FileUtils.readzip(filename) successflag = True else: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(filename + " is missing.") errorlog.append(filename + '\t' + "missing") continue tokens, pre_matched, pre_english = Volume2.as_stream(lines, verbose=debug) tokencount = len(tokens) if len(tokens) < 10: print(filename, "has only tokencount", len(tokens)) errorlog.append(filename + '\t' + 'short') correct_tokens, pages, post_matched, post_english = Volume2.correct_stream( tokens, verbose=debug) # Combine page dictionaries into a master dictionary. # If you ask, why didn't you just produce one in the first place? ... # answer has to do with flexibility of the Volume module for other purposes. pagecounter = 0 masterdict = dict() for page in pages: for item in page: if item in masterdict: masterdict[item] += page[item] else: masterdict[item] = page[item] # Now that we have a master dictionary, consider whether there are long-s problems. # This algorithm works adequately. errors = 1 truths = 1 totaladded = 0 totaldeleted = 0 # Initialized to 1 as a Laplacian correction. for word in felecterrors: errors = errors + masterdict.get(word, 0) for word in selecttruths: truths = truths + masterdict.get(word, 0) if truths > errors: LongSproblem = False else: LongSproblem = True if LongSproblem == False: corrected = correct_tokens else: longSfiles.append(filename) deleted, added, corrected, changedphrases, unchanged = Context.catch_ambiguities( correct_tokens, debug) ## Adjust wordcounts to reflect contextual spellchecking. if wordcountflag: for word, count in deleted.items(): masterdict = subtract_counts(word, masterdict, count) totaldeleted = totaldeleted + count for word, count in added.items(): masterdict = add_counts(word, masterdict, count) totaladded = totaladded + count # Write corrected file. outfilename = filename[:-suffixlen] + ".clean.txt" with open(outfilename, mode='w', encoding='utf-8') as file: lasttoken = "" for token in corrected: if lasttoken == '\n' and (token == '"' or token == "'"): token = token elif token != '\n' and token != "“" and not ( token.startswith('<') and token.endswith('>')): token = token + " " file.write(token) lasttoken = token print(outfilename) ## If we're also writing wordcount files, we need to write the .tsv file. if wordcountflag: outlist = sorted(masterdict.items(), key=lambda x: x[1], reverse=True) outfilename = outfilename[:-10] + ".vol.tsv" totalwordsinvol = 0 with open(outfilename, mode='w', encoding='utf-8') as file: for item in outlist: outline = item[0] + delim + str(item[1]) + '\n' file.write(outline) totalwordsinvol += item[1] print(outfilename) metatuple = (outfilename, str(totalwordsinvol), str(pre_matched), str(pre_english), str(post_matched), str(post_english), str(totaladded), str(totaldeleted)) processedmeta.append(metatuple) count += 1 if count > 200: break # END ITERATION ACROSS FILES. # Write the errorlog and list of long S files. errorpath = FileUtils.clearpath(rootpath, "processingerrors.txt") longSpath = FileUtils.clearpath(rootpath, "longSfiles.txt") metapath = FileUtils.clearpath(rootpath, "processing_metadata.tsv") if len(errorlog) > 0: with open(errorpath, mode='w', encoding='utf-8') as file: for line in errorlog: file.write(line + '\n') print("Writing", errorpath) if len(longSfiles) > 0: with open(longSpath, mode='w', encoding='utf-8') as file: for line in longSfiles: file.write(line + '\n') print("Writing", longSpath) if len(processedmeta) > 0: with open(metapath, mode='w', encoding='utf8') as file: file.write( 'filename\twordinvol\toriginallyindict\toriginallyenglish\tindictpostcorrection\tenglishpostcorrection\taddedbycontextmodule\tdeletedbycontextmodule\n' ) for atuple in processedmeta: outline = '\t'.join(atuple) + '\n' file.write(outline) print("Writing", metapath)
for line in filelines: line = line.rstrip() line = line.split(delim) if line[0] in HTIDs: HTIDs.discard(line[0]) processedmeta = list() errorlog = list() longSfiles = list() totaladded = dict() totaldeleted = dict() for thisID in HTIDs: filepath, postfix = FileCabinet.pairtreepath(thisID, datapath) filename = filepath + postfix + '/' + postfix + ".clean.txt" try: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(thisID + " is missing.") errorlog.append(thisID + '\t' + "missing") continue tokens, aretokensverse, pagefeatures = Volume.as_stream(lines,
for line in filelines: line = line.rstrip() line = line.split(delim) if line[0] in HTIDs: HTIDs.discard(line[0]) processedmeta = list() errorlog = list() longSfiles = list() totaladded = dict() totaldeleted = dict() for thisID in HTIDs: filepath, postfix = FileCabinet.pairtreepath(thisID, datapath) filename = filepath + postfix + '/' + postfix + ".clean.txt" try: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(thisID + " is missing.") errorlog.append(thisID + '\t' + "missing") continue tokens, aretokensverse, pagefeatures = Volume.as_stream(lines, verbose=debug)
with open(HTIDfile, encoding="utf-8") as file: HTIDlist = file.readlines() Lexicon = Dictionary.BuildLexicon(dictionarypath, debug) writename = slicename + "IND.txt" delim = '\t' BigIndex = dict() SortedIndex = list() for IDtoprocess in HTIDlist: IDtoprocess = IDtoprocess.strip() filepath, postfix = FileCabinet.pairtreepath(IDtoprocess, datapath) filename = filepath + postfix + '/' + postfix + ".txt" try: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(IDtoprocess + " is missing.") continue tokens = TokenGen.keep_hyphens(lines,Lexicon,verbose=debug)
def main(): import FileCabinet import FileUtils import Volume2 import Context import sys import os # DEFINE CONSTANTS. delim = '\t' debug = False felecterrors = ['fee', 'fea', 'fay', 'fays', 'fame', 'fell', 'funk', 'fold', 'haft', 'fat', 'fix', 'chafe', 'loft'] selecttruths = ['see', 'sea', 'say', 'says', 'same', 'sell', 'sunk', 'sold', 'hast', 'sat', 'six', 'chase', 'lost'] # Locate ourselves in the directory structure. cwd = os.getcwd() cwdparent = os.path.abspath(os.path.join(cwd, os.pardir)) # We need to find a directory called 'rulesets,' which we expect to be located # either within the working directory or adjacent to it. if os.path.isdir(os.path.join(cwd, "rulesets")): rulepath = os.path.join(cwd, "rulesets") elif os.path.isdir(os.path.join(cwdparent, "rulesets")): rulepath = os.path.join(cwdparent, "rulesets") else: user = input("Please specify a path to the ruleset directory: ") if os.path.isdir(user): rulepath = user else: print("Invalid path.") sys.exit() # Use rulepath to load relevant rules inside modules. Volume2.importrules(rulepath) Context.importrules(rulepath) # Now we enter dialogue with the user. This is all a little 1982, # but what can I say? Wetware upgrades are expensive. def prompt(promptstring, options): user = input(promptstring) if user not in options: user = prompt(promptstring, options) return user # Ask the user to tell us how to find files to process. print("****************** CorrectOCR 0.1 ******************") print() print("Do you want the full spiel (explanations, caveats, etc.)") user = prompt("y/n : ", ["y", "n"]) if user.lower() == "y": spielpath = os.path.join(cwd, "spiel.txt") with open(spielpath, encoding = 'utf-8') as file: filelines = file.readlines() for line in filelines: print(line, end='') print("\nThis script will correct .txt files, or extract text") print("from zipped archives containing one txt file for each page.") print("In either case it writes the cleaned files back to their") print("original locations with the new suffix '.clean.txt'.") print("\nDo you want to unpack .zip files or .txt files?") user = prompt("zip or txt: ", ["zip", "txt"]) suffix = "." + user suffixlen = len(suffix) print("\nThere are two ways to identify the location of the") print("files to be corrected.") print("\n1. Provide the path to a folder that contains them. I'll") print("recursively search subdirectories of that folder as well. Or,") print("\n2. Provide a file holding a list of pairtree file identifiers,") print("e.g. HathiTrust Volume IDs. I can use those identifiers to infer") print("the paths to the files themselves.\n") user = prompt("Which option do you prefer (1 or 2)? ", ["1", "2"]) if user == "1": rootpath = input("Path to the folder that contains source files: ") filelist = FileUtils.recursivefilegetter(rootpath, suffix) else: print("I expect the pairtree identifiers to be listed one per line,") print("and to be the only item on a line.") filepath = input("Path to the file that contains pairtree identifiers: ") filelist = list() with open(filepath, encoding = 'utf-8') as file: filelines = file.readlines() print("Now I need a path to the folder that contains the pairtree structure.") print("If you have multiple folders for different libraries, this should be") print("the folder above them all. It should end with a slash.") rootpath = input("Path to the folder that contains pairtree: ") for line in filelines: line = line.rstrip() filepath, postfix = FileCabinet.pairtreepath(line, rootpath) filename = filepath + postfix + '/' + postfix + suffix filelist.append(filename) print("\nI identified", len(filelist), "files in that location.") print("\nI can just write clean text files (with suffix clean.txt)") print("or I can also write tab-separated files that count the words") print("in each file after correction.") user = prompt("1) Text only or 2) text-plus-wordcounts? (1 or 2): ", ["1", "2"]) if user == "1": wordcountflag = False else: wordcountflag = True print("Now proceeding to process the files.\n") def subtract_counts (token, adict, tosubtract): '''Adjusts a dictionary by subtracting tosubtract instances of token.''' if token in adict: adict[token] = adict[token] - tosubtract if adict[token] < 0: del adict[token] elif adict[token] < 1: del adict[token] return adict def add_counts (token, adict, toadd): '''Adjusts a dictionary by adding toadd instances of token.''' if token in adict: adict[token] = adict[token] + toadd else: adict[token] = toadd return adict # Here's where we BEGIN THE ACTUAL CORRECTION OF FILES. processedmeta = list() errorlog = list() longSfiles = list() count = 0 for filename in filelist: try: if suffix == ".zip": lines = FileUtils.readzip(filename) successflag = True else: with open(filename, encoding='utf-8') as file: lines = file.readlines() successflag = True except IOError as e: successflag = False if not successflag: print(filename + " is missing.") errorlog.append(filename + '\t' + "missing") continue tokens, pre_matched, pre_english = Volume2.as_stream(lines, verbose=debug) tokencount = len(tokens) if len(tokens) < 10: print(filename, "has only tokencount", len(tokens)) errorlog.append(filename + '\t' + 'short') correct_tokens, pages, post_matched, post_english = Volume2.correct_stream(tokens, verbose = debug) # Combine page dictionaries into a master dictionary. # If you ask, why didn't you just produce one in the first place? ... # answer has to do with flexibility of the Volume module for other purposes. pagecounter = 0 masterdict = dict() for page in pages: for item in page: if item in masterdict: masterdict[item] += page[item] else: masterdict[item] = page[item] # Now that we have a master dictionary, consider whether there are long-s problems. # This algorithm works adequately. errors = 1 truths = 1 totaladded = 0 totaldeleted = 0 # Initialized to 1 as a Laplacian correction. for word in felecterrors: errors = errors + masterdict.get(word, 0) for word in selecttruths: truths = truths + masterdict.get(word, 0) if truths > errors: LongSproblem = False else: LongSproblem = True if LongSproblem == False: corrected = correct_tokens else: longSfiles.append(filename) deleted, added, corrected, changedphrases, unchanged = Context.catch_ambiguities(correct_tokens, debug) ## Adjust wordcounts to reflect contextual spellchecking. if wordcountflag: for word, count in deleted.items(): masterdict = subtract_counts(word, masterdict, count) totaldeleted = totaldeleted + count for word, count in added.items(): masterdict = add_counts(word, masterdict, count) totaladded = totaladded + count # Write corrected file. outfilename = filename[:-suffixlen] + ".clean.txt" with open(outfilename, mode = 'w', encoding = 'utf-8') as file: lasttoken = "" for token in corrected: if lasttoken == '\n' and (token == '"' or token == "'"): token = token elif token != '\n' and token != "“" and not (token.startswith('<') and token.endswith('>')): token = token + " " file.write(token) lasttoken = token print(outfilename) ## If we're also writing wordcount files, we need to write the .tsv file. if wordcountflag: outlist = sorted(masterdict.items(), key = lambda x: x[1], reverse = True) outfilename = outfilename[ :-10] + ".vol.tsv" totalwordsinvol = 0 with open(outfilename, mode = 'w', encoding = 'utf-8') as file: for item in outlist: outline = item[0] + delim + str(item[1]) + '\n' file.write(outline) totalwordsinvol += item[1] print(outfilename) metatuple = (outfilename, str(totalwordsinvol), str(pre_matched), str(pre_english), str(post_matched), str(post_english), str(totaladded), str(totaldeleted)) processedmeta.append(metatuple) count += 1 if count > 200: break # END ITERATION ACROSS FILES. # Write the errorlog and list of long S files. errorpath = FileUtils.clearpath(rootpath, "processingerrors.txt") longSpath = FileUtils.clearpath(rootpath, "longSfiles.txt") metapath = FileUtils.clearpath(rootpath, "processing_metadata.tsv") if len(errorlog) > 0: with open(errorpath, mode = 'w', encoding = 'utf-8') as file: for line in errorlog: file.write(line + '\n') print("Writing", errorpath) if len(longSfiles) > 0: with open(longSpath, mode = 'w', encoding = 'utf-8') as file: for line in longSfiles: file.write(line + '\n') print("Writing", longSpath) if len(processedmeta) > 0: with open(metapath, mode = 'w', encoding = 'utf8') as file: file.write('filename\twordinvol\toriginallyindict\toriginallyenglish\tindictpostcorrection\tenglishpostcorrection\taddedbycontextmodule\tdeletedbycontextmodule\n') for atuple in processedmeta: outline = '\t'.join(atuple) + '\n' file.write(outline) print("Writing", metapath)