def getConfirmedFilenameTuples(self, strictCheck=False): """ Return a list of tuples of UPPER CASE book codes with actual (present and readable) USX filenames. If the strictCheck flag is set, the program also looks at the first line(s) inside the files. The result is a list of 2-tuples in the default rough sequence order from the BibleBooksCodes module. Each tuple contains ( BBB, filename ) not including the folder path. """ resultList = [] for BBB, possibleFilename in self.getDerivedFilenameTuples(): possibleFilepath = os.path.join(self.givenFolderName, possibleFilename) #print( ' Looking for: ' + possibleFilename ) if os.access(possibleFilepath, os.R_OK): #print( "possibleFilepath", possibleFilepath ) #USXBookCode = possibleFilename[self.USXBookCodeIndex:self.USXBookCodeIndex+3].upper() if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile( possibleFilename, self.givenFolderName, numLines=3) #print( "firstLinesGCFT", firstLines ) if not firstLines or len(firstLines) < 3: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 3: print( "USXB (unexpected) first line was {!r} in {}". format(firstLines, thisFilename)) if '<usx' not in firstLines[ 0] and '<usx' not in firstLines[1]: continue # so it doesn't get added resultList.append(( BBB, possibleFilename, )) return resultList # No need to sort these, coz the above call produce sorted results
def getPossibleFilenameTuples( self, strictCheck=False ): """ Return a list of filenames just derived from the list of files in the folder, i.e., look only externally at the filenames. If the strictCheck flag is set, the program also looks at the first line(s) inside the files. """ #print( "getPossibleFilenameTuples()" ) resultList = [] for possibleFilename in self.fileList: pFUpper = possibleFilename.upper() if pFUpper in filenamesToIgnore: continue pFUpperProper, pFUpperExt = os.path.splitext( pFUpper ) for USFMBookCode,USFMDigits,BBB in self._USFMBooksCodeNumberTriples: ignore = False for ending in filenameEndingsToIgnore: if pFUpper.endswith( ending): ignore=True; break if ignore: continue if USFMBookCode.upper() in pFUpperProper: if pFUpper[-1]!='~' and not pFUpperExt[1:] in extensionsToIgnore: # Compare without the first dot if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile( possibleFilename, self.givenFolderName, numLines=3 ) #print( "firstLinesGPFT", firstLines ) if not firstLines or len(firstLines)<3: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print( "USXB (unexpected) first line was {!r} in {}".format( firstLines, thisFilename ) ) if '<usx' not in firstLines[0] and '<usx' not in firstLines[1]: continue # so it doesn't get added self.doListAppend( BibleOrgSysGlobals.BibleBooksCodes.getBBBFromUSFM( USFMBookCode ), possibleFilename, resultList, "getPossibleFilenameTuplesExt" ) self.lastTupleList = resultList #print( "resultList", resultList ) return BibleOrgSysGlobals.BibleBooksCodes.getSequenceList( resultList )
def getMaximumPossibleFilenameTuples( self, strictCheck=False ): """ Find the method that finds the maximum number of USFM Bible files. The result is a list of 2-tuples in the default rough sequence order from the BibleBooksCodes module. Each tuple contains ( BBB, filename ) not including the folder path. """ #if BibleOrgSysGlobals.debugFlag: print( "getMaximumPossibleFilenameTuples( {} )".format( strictCheck ) ) resultString, resultList = 'Confirmed', self.getConfirmedFilenameTuples() resultListExt = self.getPossibleFilenameTuplesExt() if len(resultListExt) > len(resultList): resultString, resultList = 'External', resultListExt resultListInt = self.getPossibleFilenameTuplesInt() if len(resultListInt) > len(resultList): resultString, resultList = 'Internal', resultListInt if BibleOrgSysGlobals.verbosityLevel > 2: print( "getMaximumPossibleFilenameTuples: using {}".format( resultString ) ) if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: #if BibleOrgSysGlobals.debugFlag: print( " getMaximumPossibleFilenameTuples doing strictCheck…" ) for BBB,filename in resultList[:]: firstLine = BibleOrgSysGlobals.peekIntoFile( filename, self.givenFolderName ) #print( 'UFN', repr(firstLine) ) if firstLine is None: resultList.remove( (BBB,filename) ); continue # seems we couldn't decode the file if firstLine and firstLine[0]==chr(65279): #U+FEFF or \ufeff logging.info( "USFMBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}".format( filename ) ) firstLine = firstLine[1:] # Remove the Unicode Byte Order Marker (BOM) if not firstLine or firstLine[0] != '\\': # don't allow a blank first line and must start with a backslash resultList.remove( (BBB,filename) ) self.lastTupleList = resultList #print( "getMaximumPossibleFilenameTuples is returning", resultList ) return resultList # No need to sort these, coz all the above calls produce sorted results
def getMaximumPossibleFilenameTuples(self, strictCheck=False): """ Find the method that finds the maximum number of USFM Bible files. The result is a list of 2-tuples in the default rough sequence order from the BibleBooksCodes module. Each tuple contains ( BBB, filename ) not including the folder path. """ #if BibleOrgSysGlobals.debugFlag: print( "getMaximumPossibleFilenameTuples( {} )".format( strictCheck ) ) resultString, resultList = 'Confirmed', self.getConfirmedFilenameTuples( ) resultListExt = self.getPossibleFilenameTuplesExt() if len(resultListExt) > len(resultList): resultString, resultList = 'External', resultListExt resultListInt = self.getPossibleFilenameTuplesInt() if len(resultListInt) > len(resultList): resultString, resultList = 'Internal', resultListInt if BibleOrgSysGlobals.verbosityLevel > 2: print("getMaximumPossibleFilenameTuples: using {}".format( resultString)) if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: #if BibleOrgSysGlobals.debugFlag: print( " getMaximumPossibleFilenameTuples doing strictCheck…" ) for BBB, filename in resultList[:]: firstLine = BibleOrgSysGlobals.peekIntoFile( filename, self.givenFolderName) #print( 'UFN', repr(firstLine) ) if firstLine is None: resultList.remove((BBB, filename)) continue # seems we couldn't decode the file if firstLine and firstLine[0] == chr(65279): #U+FEFF or \ufeff logging.info( "USFMBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}" .format(filename)) firstLine = firstLine[ 1:] # Remove the Unicode Byte Order Marker (BOM) if not firstLine or firstLine[ 0] != '\\': # don't allow a blank first line and must start with a backslash resultList.remove((BBB, filename)) self.lastTupleList = resultList #print( "getMaximumPossibleFilenameTuples is returning", resultList ) return resultList # No need to sort these, coz all the above calls produce sorted results
def UnboundBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for Unbound Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one Unbound Bible is found, returns the loaded UnboundBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck( {}, {}, {}, {} )".format(givenFolderName, strictCheck, autoLoad, autoLoadBooks)) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance(givenFolderName, str) if BibleOrgSysGlobals.debugFlag: assert autoLoad in (True, False) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical(_("UnboundBibleFileCheck: Given {!r} folder is unreadable").format(givenFolderName)) return False if not os.path.isdir(givenFolderName): logging.critical(_("UnboundBibleFileCheck: Given {!r} path is not a folder").format(givenFolderName)) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print(" UnboundBibleFileCheck: Looking for files in given {}".format(givenFolderName)) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): if something == "__MACOSX": continue # don't visit these directories foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext(somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append(something) # See if there's an UnboundBible project here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted(foundFiles): if thisFilename in ("book_names.txt", "Readme.txt"): looksHopeful = True elif thisFilename.endswith("_utf8.txt"): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile(thisFilename, givenFolderName) if firstLine is None: continue # seems we couldn't decode the file if firstLine != "#THE UNBOUND BIBLE (www.unboundbible.org)": if BibleOrgSysGlobals.verbosityLevel > 2: print("UB (unexpected) first line was {!r} in {}".format(firstLine, thisFilename)) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): uB = UnboundBible( givenFolderName, lastFilenameFound[:-9] ) # Remove the end of the actual filename "_utf8.txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print(" Looked hopeful but no actual files found") # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + "/") if not os.access(tryFolderName, os.R_OK): # The subfolder is not readable logging.warning(_("UnboundBibleFileCheck: {!r} subfolder is unreadable").format(tryFolderName)) continue if BibleOrgSysGlobals.verbosityLevel > 3: print(" UnboundBibleFileCheck: Looking for files in {}".format(tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext(somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append(something) # See if there's an UB project here in this folder for thisFilename in sorted(foundSubfiles): if thisFilename.endswith("_utf8.txt"): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile(thisFilename, tryFolderName) if firstLine is None: continue # seems we couldn't decode the file if firstLine != "#THE UNBOUND BIBLE (www.unboundbible.org)": if BibleOrgSysGlobals.verbosityLevel > 2: print("UB (unexpected) first line was {!r} in {}".format(firstLine, thisFilename)) halt continue foundProjects.append((tryFolderName, thisFilename)) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 uB = UnboundBible( foundProjects[0][0], foundProjects[0][1][:-9] ) # Remove the end of the actual filename "_utf8.txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def ForgeForSwordSearcherBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for ForgeForSwordSearcher Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one ForgeForSwordSearcher Bible is found, returns the loaded ForgeForSwordSearcherBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("ForgeForSwordSearcherBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks)) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance(givenFolderName, str) if BibleOrgSysGlobals.debugFlag: assert autoLoad in ( True, False, ) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical( _("ForgeForSwordSearcherBibleFileCheck: Given {} folder is unreadable" ).format(repr(givenFolderName))) return False if not os.path.isdir(givenFolderName): logging.critical( _("ForgeForSwordSearcherBibleFileCheck: Given {} path is not a folder" ).format(repr(givenFolderName))) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " ForgeForSwordSearcherBibleFileCheck: Looking for files in given {}" .format(repr(givenFolderName))) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): if something == '__MACOSX': continue # don't visit these directories foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append(something) # See if there's an ForgeForSwordSearcherBible project here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted(foundFiles): if thisFilename in ('book_names.txt', 'Readme.txt'): looksHopeful = True elif thisFilename.endswith('.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName) #print( '1', repr(firstLine) ) if firstLine is None: continue # seems we couldn't decode the file if firstLine and firstLine[0] == chr(65279): #U+FEFF or \ufeff logging.info( "ForgeForSwordSearcherBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}" .format(thisFilename)) firstLine = firstLine[ 1:] # Remove the Unicode Byte Order Marker (BOM) match = re.search('^; TITLE:\\s', firstLine) if match: if BibleOrgSysGlobals.debugFlag: print( "ForgeForSwordSearcherBibleFileCheck First line got {!r} match from {!r}" .format(match.group(0), firstLine)) else: if BibleOrgSysGlobals.verbosityLevel > 3: print( "ForgeForSwordSearcherBibleFileCheck: (unexpected) first line was {!r} in {}" .format(firstLine, thisFilename)) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("ForgeForSwordSearcherBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): uB = ForgeForSwordSearcherBible( givenFolderName, lastFilenameFound[:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print(" Looked hopeful but no actual files found") # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + '/') if not os.access(tryFolderName, os.R_OK): # The subfolder is not readable logging.warning( _("ForgeForSwordSearcherBibleFileCheck: {!r} subfolder is unreadable" ).format(tryFolderName)) continue if BibleOrgSysGlobals.verbosityLevel > 3: print( " ForgeForSwordSearcherBibleFileCheck: Looking for files in {}" .format(tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append(something) # See if there's an ForgeForSwordSearcherBible here in this folder for thisFilename in sorted(foundSubfiles): if thisFilename.endswith('.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName) #print( '2', repr(firstLine) ) if firstLine is None: continue # seems we couldn't decode the file if firstLine and firstLine[0] == chr( 65279): #U+FEFF or \ufeff logging.info( "ForgeForSwordSearcherBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}" .format(thisFilename)) firstLine = firstLine[ 1:] # Remove the Unicode Byte Order Marker (BOM) match = re.search('^; TITLE:\\s', firstLine) if match: if BibleOrgSysGlobals.debugFlag: print( "ForgeForSwordSearcherBibleFileCheck First line got type {!r} match from {!r}" .format(match.group(0), firstLine)) else: if BibleOrgSysGlobals.verbosityLevel > 3: print( "ForgeForSwordSearcherBibleFileCheck: (unexpected) first line was {!r} in {}" .format(firstLine, thisFilename)) if BibleOrgSysGlobals.debugFlag and debuggingThisModule: halt continue foundProjects.append(( tryFolderName, thisFilename, )) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("ForgeForSwordSearcherBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 uB = ForgeForSwordSearcherBible( foundProjects[0][0], foundProjects[0][1] [:-4]) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def YETBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for YET Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one YET Bible is found, returns the loaded YETBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("YETBibleFileCheck( {}, {}, {} )".format(givenFolderName, strictCheck, autoLoad)) if BibleOrgSysGlobals.debugFlag: assert (givenFolderName and isinstance(givenFolderName, str)) if BibleOrgSysGlobals.debugFlag: assert (autoLoad in ( True, False, )) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical( _("YETBibleFileCheck: Given {!r} folder is unreadable").format( givenFolderName)) return False if not os.path.isdir(givenFolderName): logging.critical( _("YETBibleFileCheck: Given {!r} path is not a folder").format( givenFolderName)) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print(" YETBibleFileCheck: Looking for files in given {}".format( givenFolderName)) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) if somethingUpperExt in filenameEndingsToAccept: foundFiles.append(something) if '__MACOSX' in foundFolders: foundFolders.remove('__MACOSX') # don't visit these directories # See if there's an YETBible project here in this given folder numFound = 0 lastFilenameFound = None for thisFilename in sorted(foundFiles): if thisFilename.endswith('.yet'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName) if not firstLine.startswith("info\t"): if BibleOrgSysGlobals.verbosityLevel > 2: print( "YETBible (unexpected) first line was {!r} in {}". format(firstLine, thisFilename)) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("YETBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): uB = YETBible(givenFolderName, lastFilenameFound[:-4] ) # Remove the end of the actual filename ".yet" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + '/') if not os.access(tryFolderName, os.R_OK): # The subfolder is not readable logging.warning( _("YETBibleFileCheck: {!r} subfolder is unreadable").format( tryFolderName)) continue if BibleOrgSysGlobals.verbosityLevel > 3: print(" YETBibleFileCheck: Looking for files in {}".format( tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) if somethingUpperExt in filenameEndingsToAccept: foundSubfiles.append(something) # See if there's an YETBible project here in this folder for thisFilename in sorted(foundSubfiles): if thisFilename.endswith('.yet'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName) if not firstLine.startswith("info\t"): if BibleOrgSysGlobals.verbosityLevel > 2: print( "YETBible (unexpected) first line was {!r} in {}" .format(firstLine, thisFilname)) halt continue foundProjects.append(( tryFolderName, thisFilename, )) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("YETBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert (len(foundProjects) == 1) uB = YETBible( foundProjects[0][0], foundProjects[0][1] [:-9]) # Remove the end of the actual filename "_utf8.txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def CSVBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for CSV Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one CSV Bible is found, returns the loaded CSVBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("CSVBibleFileCheck( {}, {}, {} )".format(givenFolderName, strictCheck, autoLoad)) if BibleOrgSysGlobals.debugFlag: assert (givenFolderName and isinstance(givenFolderName, str)) if BibleOrgSysGlobals.debugFlag: assert (autoLoad in ( True, False, ) and autoLoadBooks in ( True, False, )) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical( _("CSVBibleFileCheck: Given {} folder is unreadable").format( repr(givenFolderName))) return False if not os.path.isdir(givenFolderName): logging.critical( _("CSVBibleFileCheck: Given {} path is not a folder").format( repr(givenFolderName))) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print(" CSVBibleFileCheck: Looking for files in given {}".format( repr(givenFolderName))) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append(something) if '__MACOSX' in foundFolders: foundFolders.remove('__MACOSX') # don't visit these directories # See if there's an CSV Bible here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted(foundFiles): if thisFilename in ('book_names.txt', 'Readme.txt'): looksHopeful = True elif thisFilename.endswith('.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName) if firstLine is None: continue # seems we couldn't decode the file if not firstLine.startswith( '"Book","Chapter","Verse",' ) and not firstLine.startswith( '"1","1","1",') \ and not firstLine.startswith( 'Book,Chapter,Verse,' ) and not firstLine.startswith( '1,1,1,'): if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck: (unexpected) first line was {!r} in {}" .format(firstLine, thisFilename)) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("CSVBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): uB = CSVBible(givenFolderName, lastFilenameFound[:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print(" Looked hopeful but no actual files found") # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + '/') if not os.access(tryFolderName, os.R_OK): # The subfolder is not readable logging.warning( _("CSVBibleFileCheck: {!r} subfolder is unreadable").format( tryFolderName)) continue if BibleOrgSysGlobals.verbosityLevel > 3: print(" CSVBibleFileCheck: Looking for files in {}".format( tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append(something) # See if there's an CSV Bible here in this folder for thisFilename in sorted(foundSubfiles): if thisFilename.endswith('.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName) if firstLine is None: continue # seems we couldn't decode the file if not firstLine.startswith("Ge 1:1 "): if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck: (unexpected) first line was {!r} in {}" .format(firstLine, thisFilename)) if debuggingThisModule: halt continue foundProjects.append(( tryFolderName, thisFilename, )) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("CSVBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert (len(foundProjects) == 1) uB = CSVBible( foundProjects[0][0], foundProjects[0][1] [:-4]) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def UnboundBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for Unbound Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one Unbound Bible is found, returns the loaded UnboundBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks)) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance(givenFolderName, str) if BibleOrgSysGlobals.debugFlag: assert autoLoad in ( True, False, ) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical( _("UnboundBibleFileCheck: Given {!r} folder is unreadable").format( givenFolderName)) return False if not os.path.isdir(givenFolderName): logging.critical( _("UnboundBibleFileCheck: Given {!r} path is not a folder").format( givenFolderName)) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print(" UnboundBibleFileCheck: Looking for files in given {}".format( givenFolderName)) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): if something in BibleOrgSysGlobals.COMMONLY_IGNORED_FOLDERS: continue # don't visit these directories foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append(something) # See if there's an UnboundBible project here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted(foundFiles): if thisFilename in ('book_names.txt', 'Readme.txt'): looksHopeful = True elif thisFilename.endswith('_utf8.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName) if firstLine is None: continue # seems we couldn't decode the file if firstLine != "#THE UNBOUND BIBLE (www.unboundbible.org)": if BibleOrgSysGlobals.verbosityLevel > 3: print("UnB (unexpected) first line was {!r} in {}". format(firstLine, thisFilename)) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): uB = UnboundBible( givenFolderName, lastFilenameFound[:-9] ) # Remove the end of the actual filename "_utf8.txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print(" Looked hopeful but no actual files found") # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + '/') if not os.access(tryFolderName, os.R_OK): # The subfolder is not readable logging.warning( _("UnboundBibleFileCheck: {!r} subfolder is unreadable"). format(tryFolderName)) continue if BibleOrgSysGlobals.verbosityLevel > 3: print(" UnboundBibleFileCheck: Looking for files in {}".format( tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append(something) # See if there's an UB project here in this folder for thisFilename in sorted(foundSubfiles): if thisFilename.endswith('_utf8.txt'): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName) if firstLine is None: continue # seems we couldn't decode the file if firstLine != "#THE UNBOUND BIBLE (www.unboundbible.org)": if BibleOrgSysGlobals.verbosityLevel > 3: print("UnB (unexpected) first line was {!r} in {}". format(firstLine, thisFilename)) halt continue foundProjects.append(( tryFolderName, thisFilename, )) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("UnboundBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 uB = UnboundBible( foundProjects[0][0], foundProjects[0][1] [:-9]) # Remove the end of the actual filename "_utf8.txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def HaggaiXMLBibleFileCheck( givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False ): """ Given a folder, search for Haggai XML Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number found. if autoLoad is true and exactly one Haggai Bible is found, returns the loaded HaggaiXMLBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print( "HaggaiXMLBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks ) ) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance( givenFolderName, str ) if BibleOrgSysGlobals.debugFlag: assert autoLoad in (True,False,) # Check that the given folder is readable if not os.access( givenFolderName, os.R_OK ): logging.critical( _("HaggaiXMLBibleFileCheck: Given {!r} folder is unreadable").format( givenFolderName ) ) return False if not os.path.isdir( givenFolderName ): logging.critical( _("HaggaiXMLBibleFileCheck: Given {!r} path is not a folder").format( givenFolderName ) ) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " HaggaiXMLBibleFileCheck: Looking for files in given {}".format( givenFolderName ) ) foundFolders, foundFiles = [], [] for something in os.listdir( givenFolderName ): somepath = os.path.join( givenFolderName, something ) if os.path.isdir( somepath ): if something == '__MACOSX': continue # don't visit these directories foundFolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append( something ) #print( 'ff', foundFiles ) # See if there's an Haggai project here in this folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted( foundFiles ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName, numLines=2 ) if not firstLines or len(firstLines)<2: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print( "HB (unexpected) first line was {!r} in {}".format( firstLines, thisFilename ) ) continue if 'haggai_' not in firstLines[1]: continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "HaggaiXMLBibleFileCheck got", numFound, givenFolderName, lastFilenameFound ) if numFound == 1 and (autoLoad or autoLoadBooks): ub = HaggaiXMLBible( givenFolderName, lastFilenameFound ) if autoLoadBooks: ub.load() # Load and process the file return ub return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print( " Looked hopeful but no actual files found" ) # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted( foundFolders ): tryFolderName = os.path.join( givenFolderName, thisFolderName+'/' ) if BibleOrgSysGlobals.verbosityLevel > 3: print( " HaggaiXMLBibleFileCheck: Looking for files in {}".format( tryFolderName ) ) foundSubfolders, foundSubfiles = [], [] for something in os.listdir( tryFolderName ): somepath = os.path.join( givenFolderName, thisFolderName, something ) if os.path.isdir( somepath ): foundSubfolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append( something ) #print( 'fsf', foundSubfiles ) # See if there's an OS project here in this folder for thisFilename in sorted( foundSubfiles ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName, numLines=2 ) if not firstLines or len(firstLines)<2: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print( "HB (unexpected) first line was {!r} in {}".format( firstLines, thisFilename ) ) continue if 'haggai_' not in firstLines[1]: continue foundProjects.append( (tryFolderName, thisFilename,) ) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "HaggaiXMLBibleFileCheck foundProjects", numFound, foundProjects ) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 ub = HaggaiXMLBible( foundProjects[0][0], foundProjects[0][1] ) # Folder and filename if autoLoadBooks: ub.load() # Load and process the file return ub return numFound
def OpenSongXMLBibleFileCheck(givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False): """ Given a folder, search for OpenSong XML Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number found. if autoLoad is true and exactly one OpenSong Bible is found, returns the loaded OpenSongXMLBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print("OpenSongXMLBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks)) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance(givenFolderName, str) if BibleOrgSysGlobals.debugFlag: assert autoLoad in ( True, False, ) if BibleOrgSysGlobals.debugFlag: assert autoLoadBooks in ( True, False, ) # Check that the given folder is readable if not os.access(givenFolderName, os.R_OK): logging.critical( _("OpenSongXMLBibleFileCheck: Given {!r} folder is unreadable"). format(givenFolderName)) return False if not os.path.isdir(givenFolderName): logging.critical( _("OpenSongXMLBibleFileCheck: Given {!r} path is not a folder"). format(givenFolderName)) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " OpenSongXMLBibleFileCheck: Looking for files in given {}".format( givenFolderName)) foundFolders, foundFiles = [], [] for something in os.listdir(givenFolderName): somepath = os.path.join(givenFolderName, something) if os.path.isdir(somepath): if something == '__MACOSX': continue # don't visit these directories foundFolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append(something) #print( 'osx1', foundFiles ) # See if there's an OpenSong project here in this folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted(foundFiles): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile(thisFilename, givenFolderName, numLines=2) #print( 'osx1b', firstLines ) if not firstLines or len(firstLines) < 2: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print("OSB (unexpected) first line was {!r} in {}".format( firstLines, thisFilename)) continue if not firstLines[1].startswith('<bible>'): continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("OpenSongXMLBibleFileCheck got", numFound, givenFolderName, lastFilenameFound) if numFound == 1 and (autoLoad or autoLoadBooks): osb = OpenSongXMLBible(givenFolderName, lastFilenameFound) if autoLoadBooks: osb.load() # Load and process the file return osb return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print(" Looked hopeful but no actual files found") # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted(foundFolders): tryFolderName = os.path.join(givenFolderName, thisFolderName + '/') if BibleOrgSysGlobals.verbosityLevel > 3: print(" OpenSongXMLBibleFileCheck: Looking for files in {}". format(tryFolderName)) foundSubfolders, foundSubfiles = [], [] for something in os.listdir(tryFolderName): somepath = os.path.join(givenFolderName, thisFolderName, something) if os.path.isdir(somepath): foundSubfolders.append(something) elif os.path.isfile(somepath): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith(ending): ignore = True break if ignore: continue if not somethingUpperExt[ 1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append(something) #print( 'osx2', foundSubfiles ) # See if there's an OS project here in this folder for thisFilename in sorted(foundSubfiles): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLines = BibleOrgSysGlobals.peekIntoFile(thisFilename, tryFolderName, numLines=2) if not firstLines or len(firstLines) < 2: continue if not ( firstLines[0].startswith( '<?xml version="1.0"' ) or firstLines[0].startswith( "<?xml version='1.0'" ) ) \ and not ( firstLines[0].startswith( '\ufeff<?xml version="1.0"' ) or firstLines[0].startswith( "\ufeff<?xml version='1.0'" ) ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print("OSB (unexpected) first line was {!r} in {}". format(firstLines, thisFilename)) continue if not firstLines[1].startswith('<bible>'): continue foundProjects.append(( tryFolderName, thisFilename, )) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print("OpenSongXMLBibleFileCheck foundProjects", numFound, foundProjects) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 osb = OpenSongXMLBible(foundProjects[0][0], foundProjects[0][1]) # Folder and filename if autoLoadBooks: osb.load() # Load and process the file return osb return numFound
def DrupalBibleFileCheck( givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False ): """ Given a folder, search for DrupalBible Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one DrupalBible Bible is found, returns the loaded DrupalBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print( "DrupalBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks ) ) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance( givenFolderName, str ) if BibleOrgSysGlobals.debugFlag: assert autoLoad in (True,False,) # Check that the given folder is readable if not os.access( givenFolderName, os.R_OK ): logging.critical( _("DrupalBibleFileCheck: Given {!r} folder is unreadable").format( givenFolderName ) ) return False if not os.path.isdir( givenFolderName ): logging.critical( _("DrupalBibleFileCheck: Given {!r} path is not a folder").format( givenFolderName ) ) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " DrupalBibleFileCheck: Looking for files in given {}".format( givenFolderName ) ) foundFolders, foundFiles = [], [] for something in os.listdir( givenFolderName ): somepath = os.path.join( givenFolderName, something ) if os.path.isdir( somepath ): if something == '__MACOSX': continue # don't visit these directories foundFolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) if somethingUpperExt in filenameEndingsToAccept: foundFiles.append( something ) # See if there's an DrupalBible project here in this given folder numFound = 0 lastFilenameFound = None for thisFilename in sorted( foundFiles ): if thisFilename.endswith( '.bc' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName ) if firstLine is None: continue # seems we couldn't decode the file if ( not firstLine.startswith( '\ufeff*Bible' ) ) and ( not firstLine.startswith( "*Bible" ) ): if BibleOrgSysGlobals.verbosityLevel > 3: print( "DrupalBible (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "DrupalBibleFileCheck got", numFound, givenFolderName, lastFilenameFound ) if numFound == 1 and (autoLoad or autoLoadBooks): uB = DrupalBible( givenFolderName, lastFilenameFound[:-3] ) # Remove the end of the actual filename ".bc" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted( foundFolders ): tryFolderName = os.path.join( givenFolderName, thisFolderName+'/' ) if not os.access( tryFolderName, os.R_OK ): # The subfolder is not readable logging.warning( _("DrupalBibleFileCheck: {!r} subfolder is unreadable").format( tryFolderName ) ) continue if BibleOrgSysGlobals.verbosityLevel > 3: print( " DrupalBibleFileCheck: Looking for files in {}".format( tryFolderName ) ) foundSubfolders, foundSubfiles = [], [] for something in os.listdir( tryFolderName ): somepath = os.path.join( givenFolderName, thisFolderName, something ) if os.path.isdir( somepath ): foundSubfolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) if somethingUpperExt in filenameEndingsToAccept: foundSubfiles.append( something ) # See if there's an DrupalBible project here in this folder for thisFilename in sorted( foundSubfiles ): if thisFilename.endswith( '.bc' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName ) if firstLine is None: continue # seems we couldn't decode the file if ( not firstLine.startswith( '\ufeff*Bible' ) ) and ( not firstLine.startswith( "*Bible" ) ): if BibleOrgSysGlobals.verbosityLevel > 3: print( "DrupalBible (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ); halt continue #print( "BFC_here", repr(tryFolderName), repr(thisFilename) ) foundProjects.append( (tryFolderName, thisFilename,) ) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "DrupalBibleFileCheck foundProjects", numFound, foundProjects ) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 uB = DrupalBible( foundProjects[0][0], foundProjects[0][1][:-3] ) # Remove the end of the actual filename ".bc" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def __init__( self, sourceFolder, givenName=None, encoding='utf-8' ): """ Create the internal USFX Bible object. """ # Setup and initialise the base class first Bible.__init__( self ) self.objectNameString = "USFX XML Bible object" self.objectTypeString = "USFX" self.sourceFolder, self.givenName, self.encoding = sourceFolder, givenName, encoding # Remember our parameters # Now we can set our object variables self.name = self.givenName if not self.name: self.name = os.path.basename( self.sourceFolder ) if not self.name: self.name = os.path.basename( self.sourceFolder[:-1] ) # Remove the final slash if not self.name: self.name = "USFX Bible" if self.name.endswith( '_usfx' ): self.name = self.name[:-5] # Remove end of name for Haiola projects # Do a preliminary check on the readability of our folder if not os.access( self.sourceFolder, os.R_OK ): logging.error( "USFXXMLBible: Folder {!r} is unreadable".format( self.sourceFolder ) ) # Do a preliminary check on the contents of our folder self.sourceFilename = self.sourceFilepath = None foundFiles, foundFolders = [], [] for something in os.listdir( self.sourceFolder ): somepath = os.path.join( self.sourceFolder, something ) if os.path.isdir( somepath ): foundFolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append( something ) else: logging.error( "Not sure what {!r} is in {}!".format( somepath, self.sourceFolder ) ) if foundFolders: logging.info( "USFXXMLBible: Surprised to see subfolders in {!r}: {}".format( self.sourceFolder, foundFolders ) ) if not foundFiles: if BibleOrgSysGlobals.verbosityLevel > 0: print( "USFXXMLBible: Couldn't find any files in {!r}".format( self.sourceFolder ) ) return # No use continuing #print( self.sourceFolder, foundFolders, len(foundFiles), foundFiles ) numFound = 0 for thisFilename in sorted( foundFiles ): firstLines = BibleOrgSysGlobals.peekIntoFile( thisFilename, sourceFolder, numLines=3 ) if not firstLines or len(firstLines)<2: continue if not firstLines[0].startswith( '<?xml version="1.0"' ) \ and not firstLines[0].startswith( '\ufeff<?xml version="1.0"' ): # same but with BOM if BibleOrgSysGlobals.verbosityLevel > 2: print( "USFXB (unexpected) first line was {!r} in {}".format( firstLines, thisFilename ) ) continue if "<usfx " not in firstLines[0]: continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "USFXXMLBible got", numFound, sourceFolder, lastFilenameFound ) if numFound == 1: self.sourceFilename = lastFilenameFound self.sourceFilepath = os.path.join( self.sourceFolder, self.sourceFilename ) elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print( " Looked hopeful but no actual files found" )
def CSVBibleFileCheck( givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False ): """ Given a folder, search for CSV Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one CSV Bible is found, returns the loaded CSVBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck( {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad ) ) if BibleOrgSysGlobals.debugFlag: assert( givenFolderName and isinstance( givenFolderName, str ) ) if BibleOrgSysGlobals.debugFlag: assert( autoLoad in (True,False,) and autoLoadBooks in (True,False,) ) # Check that the given folder is readable if not os.access( givenFolderName, os.R_OK ): logging.critical( _("CSVBibleFileCheck: Given {} folder is unreadable").format( repr(givenFolderName) ) ) return False if not os.path.isdir( givenFolderName ): logging.critical( _("CSVBibleFileCheck: Given {} path is not a folder").format( repr(givenFolderName) ) ) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " CSVBibleFileCheck: Looking for files in given {}".format( repr(givenFolderName) ) ) foundFolders, foundFiles = [], [] for something in os.listdir( givenFolderName ): somepath = os.path.join( givenFolderName, something ) if os.path.isdir( somepath ): foundFolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append( something ) if '__MACOSX' in foundFolders: foundFolders.remove( '__MACOSX' ) # don't visit these directories # See if there's an CSV Bible here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted( foundFiles ): if thisFilename in ('book_names.txt','Readme.txt' ): looksHopeful = True elif thisFilename.endswith( '.txt' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName ) if firstLine is None: continue # seems we couldn't decode the file if not firstLine.startswith( '"Book","Chapter","Verse",' ) and not firstLine.startswith( '"1","1","1",') \ and not firstLine.startswith( 'Book,Chapter,Verse,' ) and not firstLine.startswith( '1,1,1,'): if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck: (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck got", numFound, givenFolderName, lastFilenameFound ) if numFound == 1 and (autoLoad or autoLoadBooks): uB = CSVBible( givenFolderName, lastFilenameFound[:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print( " Looked hopeful but no actual files found" ) # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted( foundFolders ): tryFolderName = os.path.join( givenFolderName, thisFolderName+'/' ) if not os.access( tryFolderName, os.R_OK ): # The subfolder is not readable logging.warning( _("CSVBibleFileCheck: {!r} subfolder is unreadable").format( tryFolderName ) ) continue if BibleOrgSysGlobals.verbosityLevel > 3: print( " CSVBibleFileCheck: Looking for files in {}".format( tryFolderName ) ) foundSubfolders, foundSubfiles = [], [] for something in os.listdir( tryFolderName ): somepath = os.path.join( givenFolderName, thisFolderName, something ) if os.path.isdir( somepath ): foundSubfolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append( something ) # See if there's an CSV Bible here in this folder for thisFilename in sorted( foundSubfiles ): if thisFilename.endswith( '.txt' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName ) if firstLine is None: continue # seems we couldn't decode the file if not firstLine.startswith( "Ge 1:1 " ): if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck: (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ) if debuggingThisModule: halt continue foundProjects.append( (tryFolderName, thisFilename,) ) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "CSVBibleFileCheck foundProjects", numFound, foundProjects ) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert( len(foundProjects) == 1 ) uB = CSVBible( foundProjects[0][0], foundProjects[0][1][:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound
def ForgeForSwordSearcherBibleFileCheck( givenFolderName, strictCheck=True, autoLoad=False, autoLoadBooks=False ): """ Given a folder, search for ForgeForSwordSearcher Bible files or folders in the folder and in the next level down. Returns False if an error is found. if autoLoad is false (default) returns None, or the number of Bibles found. if autoLoad is true and exactly one ForgeForSwordSearcher Bible is found, returns the loaded ForgeForSwordSearcherBible object. """ if BibleOrgSysGlobals.verbosityLevel > 2: print( "ForgeForSwordSearcherBibleFileCheck( {}, {}, {}, {} )".format( givenFolderName, strictCheck, autoLoad, autoLoadBooks ) ) if BibleOrgSysGlobals.debugFlag: assert givenFolderName and isinstance( givenFolderName, str ) if BibleOrgSysGlobals.debugFlag: assert autoLoad in (True,False,) # Check that the given folder is readable if not os.access( givenFolderName, os.R_OK ): logging.critical( _("ForgeForSwordSearcherBibleFileCheck: Given {} folder is unreadable").format( repr(givenFolderName) ) ) return False if not os.path.isdir( givenFolderName ): logging.critical( _("ForgeForSwordSearcherBibleFileCheck: Given {} path is not a folder").format( repr(givenFolderName) ) ) return False # Find all the files and folders in this folder if BibleOrgSysGlobals.verbosityLevel > 3: print( " ForgeForSwordSearcherBibleFileCheck: Looking for files in given {}".format( repr(givenFolderName) ) ) foundFolders, foundFiles = [], [] for something in os.listdir( givenFolderName ): somepath = os.path.join( givenFolderName, something ) if os.path.isdir( somepath ): if something == '__MACOSX': continue # don't visit these directories foundFolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundFiles.append( something ) # See if there's an ForgeForSwordSearcherBible project here in this given folder numFound = 0 looksHopeful = False lastFilenameFound = None for thisFilename in sorted( foundFiles ): if thisFilename in ('book_names.txt','Readme.txt' ): looksHopeful = True elif thisFilename.endswith( '.txt' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, givenFolderName ) #print( '1', repr(firstLine) ) if firstLine is None: continue # seems we couldn't decode the file if firstLine and firstLine[0]==chr(65279): #U+FEFF or \ufeff logging.info( "ForgeForSwordSearcherBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}".format( thisFilename ) ) firstLine = firstLine[1:] # Remove the Unicode Byte Order Marker (BOM) match = re.search( '^; TITLE:\\s', firstLine ) if match: if BibleOrgSysGlobals.debugFlag: print( "ForgeForSwordSearcherBibleFileCheck First line got {!r} match from {!r}".format( match.group(0), firstLine ) ) else: if BibleOrgSysGlobals.verbosityLevel > 2: print( "ForgeForSwordSearcherBibleFileCheck: (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ) continue lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "ForgeForSwordSearcherBibleFileCheck got", numFound, givenFolderName, lastFilenameFound ) if numFound == 1 and (autoLoad or autoLoadBooks): uB = ForgeForSwordSearcherBible( givenFolderName, lastFilenameFound[:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2: print( " Looked hopeful but no actual files found" ) # Look one level down numFound = 0 foundProjects = [] for thisFolderName in sorted( foundFolders ): tryFolderName = os.path.join( givenFolderName, thisFolderName+'/' ) if not os.access( tryFolderName, os.R_OK ): # The subfolder is not readable logging.warning( _("ForgeForSwordSearcherBibleFileCheck: {!r} subfolder is unreadable").format( tryFolderName ) ) continue if BibleOrgSysGlobals.verbosityLevel > 3: print( " ForgeForSwordSearcherBibleFileCheck: Looking for files in {}".format( tryFolderName ) ) foundSubfolders, foundSubfiles = [], [] for something in os.listdir( tryFolderName ): somepath = os.path.join( givenFolderName, thisFolderName, something ) if os.path.isdir( somepath ): foundSubfolders.append( something ) elif os.path.isfile( somepath ): somethingUpper = something.upper() somethingUpperProper, somethingUpperExt = os.path.splitext( somethingUpper ) ignore = False for ending in filenameEndingsToIgnore: if somethingUpper.endswith( ending): ignore=True; break if ignore: continue if not somethingUpperExt[1:] in extensionsToIgnore: # Compare without the first dot foundSubfiles.append( something ) # See if there's an ForgeForSwordSearcherBible here in this folder for thisFilename in sorted( foundSubfiles ): if thisFilename.endswith( '.txt' ): if strictCheck or BibleOrgSysGlobals.strictCheckingFlag: firstLine = BibleOrgSysGlobals.peekIntoFile( thisFilename, tryFolderName ) #print( '2', repr(firstLine) ) if firstLine is None: continue # seems we couldn't decode the file if firstLine and firstLine[0]==chr(65279): #U+FEFF or \ufeff logging.info( "ForgeForSwordSearcherBibleFileCheck: Detected Unicode Byte Order Marker (BOM) in {}".format( thisFilename ) ) firstLine = firstLine[1:] # Remove the Unicode Byte Order Marker (BOM) match = re.search( '^; TITLE:\\s', firstLine ) if match: if BibleOrgSysGlobals.debugFlag: print( "ForgeForSwordSearcherBibleFileCheck First line got type {!r} match from {!r}".format( match.group(0), firstLine ) ) else: if BibleOrgSysGlobals.verbosityLevel > 2: print( "ForgeForSwordSearcherBibleFileCheck: (unexpected) first line was {!r} in {}".format( firstLine, thisFilename ) ) if BibleOrgSysGlobals.debugFlag and debuggingThisModule: halt continue foundProjects.append( (tryFolderName, thisFilename,) ) lastFilenameFound = thisFilename numFound += 1 if numFound: if BibleOrgSysGlobals.verbosityLevel > 2: print( "ForgeForSwordSearcherBibleFileCheck foundProjects", numFound, foundProjects ) if numFound == 1 and (autoLoad or autoLoadBooks): if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1 uB = ForgeForSwordSearcherBible( foundProjects[0][0], foundProjects[0][1][:-4] ) # Remove the end of the actual filename ".txt" if autoLoadBooks: uB.load() # Load and process the file return uB return numFound