def getChangeLogFilepath( loggingFolder, projectName ): """ """ return os.path.join( loggingFolder, \ BibleOrgSysGlobals.makeSafeFilename( projectName.replace(' ','_') + '_ChangeLog.txt' ) )
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion ) if 1: # Load and process some of our test versions count = 0 for name, abbreviation, testFolder in ( # name, abbreviation, folder ("Open English Translation—Literal Version", "OET-LV", "../../../../../Data/Work/Matigsalug/Bible/OET-LV/",), #("Matigsalug", "MBTV", "../../../../../Data/Work/Matigsalug/Bible/MBTV/",), #("ESFM Test 1", "OET-LV", "Tests/DataFilesForTests/ESFMTest1/"), #("ESFM Test 2", "OET-RV", "Tests/DataFilesForTests/ESFMTest2/"), #("All Markers Project", "WEB+", "Tests/DataFilesForTests/USFMAllMarkersProject/"), #("USFM Error Project", "UEP", "Tests/DataFilesForTests/USFMErrorProject/"), #("BOS Exported Files", "Exported", "Tests/BOS_USFM_Export/"), ): count += 1 if os.access( testFolder, os.R_OK ): if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nESFM A{}/".format( count ) ) EsfmB = ESFMBible( testFolder, name, abbreviation ) EsfmB.load() print( "Gen assumed book name:", repr( EsfmB.getAssumedBookName( 'GEN' ) ) ) print( "Gen long TOC book name:", repr( EsfmB.getLongTOCName( 'GEN' ) ) ) print( "Gen short TOC book name:", repr( EsfmB.getShortTOCName( 'GEN' ) ) ) print( "Gen book abbreviation:", repr( EsfmB.getBooknameAbbreviation( 'GEN' ) ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( EsfmB ) if BibleOrgSysGlobals.strictCheckingFlag: EsfmB.check() #print( EsfmB.books['GEN']._processedLines[0:40] ) EsfmBErrors = EsfmB.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineOptions.export: ##EsfmB.toDrupalBible() EsfmB.doAllExports( wantPhotoBible=False, wantODFs=True, wantPDFs=True ) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(abbreviation) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj ) else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) ) if 0: # Test a whole folder full of folders of ESFM Bibles testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/" def findInfo( somepath ): """ Find out info about the project from the included copyright.htm file """ cFilepath = os.path.join( somepath, "copyright.htm" ) if not os.path.exists( cFilepath ): return with open( cFilepath, encoding='utf-8' ) as myFile: # Automatically closes the file when done lastLine, lineCount = None, 0 title, nameDict = None, {} for line in myFile: lineCount += 1 if lineCount==1 and line and line[0]==chr(65279): #U+FEFF logging.info( "ESFMBible: Detected UTF-16 Byte Order Marker in copyright.htm file" ) line = line[1:] # Remove the UTF-8 Byte Order Marker if line[-1]=='\n': line = line[:-1] # Removing trailing newline character if not line: continue # Just discard blank lines lastLine = line if line.startswith("<title>"): title = line.replace("<title>","").replace("</title>","").strip() if line.startswith('<option value="'): adjLine = line.replace('<option value="','').replace('</option>','') ESFM_BBB, name = adjLine[:3], adjLine[11:] BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromESFM( ESFM_BBB ) #print( ESFM_BBB, BBB, name ) nameDict[BBB] = name return title, nameDict # end of findInfo count = totalBooks = 0 if os.access( testBaseFolder, os.R_OK ): # check that we can read the test data for something in sorted( os.listdir( testBaseFolder ) ): somepath = os.path.join( testBaseFolder, something ) if os.path.isfile( somepath ): print( "Ignoring file {!r} in {!r}".format( something, testBaseFolder ) ) elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a ESFM (partial) Bible #if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules count += 1 title = None findInfoResult = findInfo( somepath ) if findInfoResult: title, bookNameDict = findInfoResult if title is None: title = something[:-5] if something.endswith("_usfm") else something name, testFolder = title, somepath if os.access( testFolder, os.R_OK ): if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nESFM B{}/".format( count ) ) EsfmB = ESFMBible( testFolder, name ) EsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 0: print( EsfmB ) if BibleOrgSysGlobals.strictCheckingFlag: EsfmB.check() EsfmBErrors = EsfmB.getErrors() #print( EsfmBErrors ) if BibleOrgSysGlobals.commandLineOptions.export: EsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) ) if count: print( "\n{} total ESFM (partial) Bibles processed.".format( count ) ) if totalBooks: print( "{} total books ({} average per folder)".format( totalBooks, round(totalBooks/count) ) ) else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testBaseFolder ) )
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion) if 1: # Load and process some of our test versions count = 0 for name, abbreviation, testFolder in ( # name, abbreviation, folder ( "Open English Translation—Literal Version", "OET-LV", "../../../../../Data/Work/Matigsalug/Bible/OET-LV/", ), #("Matigsalug", "MBTV", "../../../../../Data/Work/Matigsalug/Bible/MBTV/",), #("ESFM Test 1", "OET-LV", "Tests/DataFilesForTests/ESFMTest1/"), #("ESFM Test 2", "OET-RV", "Tests/DataFilesForTests/ESFMTest2/"), #("All Markers Project", "WEB+", "Tests/DataFilesForTests/USFMAllMarkersProject/"), #("USFM Error Project", "UEP", "Tests/DataFilesForTests/USFMErrorProject/"), #("BOS Exported Files", "Exported", "Tests/BOS_USFM_Export/"), ): count += 1 if os.access(testFolder, os.R_OK): if BibleOrgSysGlobals.verbosityLevel > 0: print("\nESFM A{}/".format(count)) EsfmB = ESFMBible(testFolder, name, abbreviation) EsfmB.load() print("Gen assumed book name:", repr(EsfmB.getAssumedBookName('GEN'))) print("Gen long TOC book name:", repr(EsfmB.getLongTOCName('GEN'))) print("Gen short TOC book name:", repr(EsfmB.getShortTOCName('GEN'))) print("Gen book abbreviation:", repr(EsfmB.getBooknameAbbreviation('GEN'))) if BibleOrgSysGlobals.verbosityLevel > 0: print(EsfmB) if BibleOrgSysGlobals.strictCheckingFlag: EsfmB.check() #print( EsfmB.books['GEN']._processedLines[0:40] ) EsfmBErrors = EsfmB.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineOptions.export: ##EsfmB.toDrupalBible() EsfmB.doAllExports(wantPhotoBible=False, wantODFs=True, wantPDFs=True) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(abbreviation) + '.pickle', os.path.join("OutputFiles/", "BOS_Bible_Object_Pickle/")) if BibleOrgSysGlobals.verbosityLevel > 0: print("newObj is", newObj) else: print( "\nSorry, test folder {!r} is not readable on this computer." .format(testFolder)) if 0: # Test a whole folder full of folders of ESFM Bibles testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/" def findInfo(somepath): """ Find out info about the project from the included copyright.htm file """ cFilepath = os.path.join(somepath, "copyright.htm") if not os.path.exists(cFilepath): return with open(cFilepath, encoding='utf-8' ) as myFile: # Automatically closes the file when done lastLine, lineCount = None, 0 title, nameDict = None, {} for line in myFile: lineCount += 1 if lineCount == 1 and line and line[0] == chr( 65279): #U+FEFF logging.info( "ESFMBible: Detected UTF-16 Byte Order Marker in copyright.htm file" ) line = line[1:] # Remove the UTF-8 Byte Order Marker if line[-1] == '\n': line = line[:-1] # Removing trailing newline character if not line: continue # Just discard blank lines lastLine = line if line.startswith("<title>"): title = line.replace("<title>", "").replace("</title>", "").strip() if line.startswith('<option value="'): adjLine = line.replace('<option value="', '').replace('</option>', '') ESFM_BBB, name = adjLine[:3], adjLine[11:] BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromESFM( ESFM_BBB) #print( ESFM_BBB, BBB, name ) nameDict[BBB] = name return title, nameDict # end of findInfo count = totalBooks = 0 if os.access(testBaseFolder, os.R_OK): # check that we can read the test data for something in sorted(os.listdir(testBaseFolder)): somepath = os.path.join(testBaseFolder, something) if os.path.isfile(somepath): print("Ignoring file {!r} in {!r}".format( something, testBaseFolder)) elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a ESFM (partial) Bible #if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules count += 1 title = None findInfoResult = findInfo(somepath) if findInfoResult: title, bookNameDict = findInfoResult if title is None: title = something[:-5] if something.endswith( "_usfm") else something name, testFolder = title, somepath if os.access(testFolder, os.R_OK): if BibleOrgSysGlobals.verbosityLevel > 0: print("\nESFM B{}/".format(count)) EsfmB = ESFMBible(testFolder, name) EsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 0: print(EsfmB) if BibleOrgSysGlobals.strictCheckingFlag: EsfmB.check() EsfmBErrors = EsfmB.getErrors() #print( EsfmBErrors ) if BibleOrgSysGlobals.commandLineOptions.export: EsfmB.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) else: print( "\nSorry, test folder {!r} is not readable on this computer." .format(testFolder)) if count: print("\n{} total ESFM (partial) Bibles processed.".format( count)) if totalBooks: print("{} total books ({} average per folder)".format( totalBooks, round(totalBooks / count))) else: print( "\nSorry, test folder {!r} is not readable on this computer.". format(testBaseFolder))
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion ) #testFolder = "Tests/DataFilesForTests/BCVTest1/" testFolder = "OutputFiles/BOS_BCV_Export/" if 1: # demo the file checking code -- first with the whole folder and then with only one folder if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA1" ) result1 = BCVBibleFileCheck( testFolder ) if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA1", result1 ) if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA2" ) result2 = BCVBibleFileCheck( testFolder, autoLoad=True ) # But doesn't preload books if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA2", result2 ) #result2.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) ) if BibleOrgSysGlobals.strictCheckingFlag: result2.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result2.getErrors() # print( bibleErrors ) #if BibleOrgSysGlobals.commandLineArguments.export: ###result2.toDrupalBible() #result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA3" ) result3 = BCVBibleFileCheck( testFolder, autoLoad=True, autoLoadBooks=True ) if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA3", result3 ) #result3.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) ) if BibleOrgSysGlobals.strictCheckingFlag: result3.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result3.getErrors() # print( bibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##result3.toDrupalBible() result3.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) if 0: # all discovered modules in the test folder foundFolders, foundFiles = [], [] for something in os.listdir( testFolder ): somepath = os.path.join( testFolder, something ) if os.path.isdir( somepath ): foundFolders.append( something ) elif os.path.isfile( somepath ): foundFiles.append( something ) if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work if BibleOrgSysGlobals.verbosityLevel > 1: print( "\nTrying all {} discovered modules…".format( len(foundFolders) ) ) parameters = [folderName for folderName in sorted(foundFolders)] BibleOrgSysGlobals.alreadyMultiprocessing = True with multiprocessing.Pool( processes=BibleOrgSysGlobals.maxProcesses ) as pool: # start worker processes results = pool.map( testBCV, parameters ) # have the pool do our loads assert len(results) == len(parameters) # Results (all None) are actually irrelevant to us here BibleOrgSysGlobals.alreadyMultiprocessing = False else: # Just single threaded for j, someFolder in enumerate( sorted( foundFolders ) ): if BibleOrgSysGlobals.verbosityLevel > 1: print( "\nBCV D{}/ Trying {}".format( j+1, someFolder ) ) #myTestFolder = os.path.join( testFolder, someFolder+'/' ) testBCV( someFolder ) if 0: # Load and process some of our test versions count = 0 for name, encoding, testFolder in ( ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest1/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest2/"), ("Exported", 'utf-8', "Tests/BOS_BCV_Export/"), ): count += 1 if os.access( testFolder, os.R_OK ): if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV A{}/".format( count ) ) bcvB = BCVBible( testFolder, name, encoding=encoding ) bcvB.load() if BibleOrgSysGlobals.verbosityLevel > 1: print( "Gen assumed book name:", repr( bcvB.getAssumedBookName( 'GEN' ) ) ) print( "Gen long TOC book name:", repr( bcvB.getLongTOCName( 'GEN' ) ) ) print( "Gen short TOC book name:", repr( bcvB.getShortTOCName( 'GEN' ) ) ) print( "Gen book abbreviation:", repr( bcvB.getBooknameAbbreviation( 'GEN' ) ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( bcvB ) if BibleOrgSysGlobals.strictCheckingFlag: bcvB.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bcbibleErrors = bcvB.getErrors() # print( bcbibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##bcvB.toDrupalBible() bcvB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj ) else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion) #testFolder = "Tests/DataFilesForTests/BCVTest1/" testFolder = "OutputFiles/BOS_BCV_Export/" if 1: # demo the file checking code -- first with the whole folder and then with only one folder if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA1") result1 = BCVBibleFileCheck(testFolder) if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA1", result1) if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA2") result2 = BCVBibleFileCheck(testFolder, autoLoad=True) # But doesn't preload books if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA2", result2) #result2.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) ) if BibleOrgSysGlobals.strictCheckingFlag: result2.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result2.getErrors() # print( bibleErrors ) #if BibleOrgSysGlobals.commandLineArguments.export: ###result2.toDrupalBible() #result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA3") result3 = BCVBibleFileCheck(testFolder, autoLoad=True, autoLoadBooks=True) if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA3", result3) #result3.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) ) if BibleOrgSysGlobals.strictCheckingFlag: result3.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result3.getErrors() # print( bibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##result3.toDrupalBible() result3.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) if 0: # all discovered modules in the test folder foundFolders, foundFiles = [], [] for something in os.listdir(testFolder): somepath = os.path.join(testFolder, something) if os.path.isdir(somepath): foundFolders.append(something) elif os.path.isfile(somepath): foundFiles.append(something) if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work if BibleOrgSysGlobals.verbosityLevel > 1: print("\nTrying all {} discovered modules…".format( len(foundFolders))) parameters = [folderName for folderName in sorted(foundFolders)] BibleOrgSysGlobals.alreadyMultiprocessing = True with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses ) as pool: # start worker processes results = pool.map(testBCV, parameters) # have the pool do our loads assert len(results) == len( parameters ) # Results (all None) are actually irrelevant to us here BibleOrgSysGlobals.alreadyMultiprocessing = False else: # Just single threaded for j, someFolder in enumerate(sorted(foundFolders)): if BibleOrgSysGlobals.verbosityLevel > 1: print("\nBCV D{}/ Trying {}".format(j + 1, someFolder)) #myTestFolder = os.path.join( testFolder, someFolder+'/' ) testBCV(someFolder) if 0: # Load and process some of our test versions count = 0 for name, encoding, testFolder in ( ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest1/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest2/"), ("Exported", 'utf-8', "Tests/BOS_BCV_Export/"), ): count += 1 if os.access(testFolder, os.R_OK): if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV A{}/".format(count)) bcvB = BCVBible(testFolder, name, encoding=encoding) bcvB.load() if BibleOrgSysGlobals.verbosityLevel > 1: print("Gen assumed book name:", repr(bcvB.getAssumedBookName('GEN'))) print("Gen long TOC book name:", repr(bcvB.getLongTOCName('GEN'))) print("Gen short TOC book name:", repr(bcvB.getShortTOCName('GEN'))) print("Gen book abbreviation:", repr(bcvB.getBooknameAbbreviation('GEN'))) if BibleOrgSysGlobals.verbosityLevel > 0: print(bcvB) if BibleOrgSysGlobals.strictCheckingFlag: bcvB.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) bcbibleErrors = bcvB.getErrors() # print( bcbibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##bcvB.toDrupalBible() bcvB.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join("OutputFiles/", "BOS_Bible_Object_Pickle/")) if BibleOrgSysGlobals.verbosityLevel > 0: print("newObj is", newObj) else: print( "\nSorry, test folder {!r} is not readable on this computer." .format(testFolder))
def createOpenSongXML(BibleObject, outputFolder=None, controlDict=None, validationSchema=None): """ Using settings from the given control file, converts the USFM information to a UTF-8 OpenSong XML file. This format is roughly documented at http://de.wikipedia.org/wiki/OpenSong_XML but more fields can be discovered by looking at downloaded files. """ if BibleOrgSysGlobals.verbosityLevel > 1: print("Running createOpenSongXML…") if BibleOrgSysGlobals.debugFlag: assert BibleObject.books ignoredMarkers, unhandledMarkers, unhandledBooks = set(), set(), [] def writeOpenSongBook(writerObject, BBB, bkData): """Writes a book to the OpenSong XML writerObject.""" #print( 'BIBLEBOOK', [('bnumber',BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber(BBB)), ('bname',BibleOrgSysGlobals.BibleBooksCodes.getEnglishName_NR(BBB)), ('bsname',BibleOrgSysGlobals.BibleBooksCodes.getOSISAbbreviation(BBB))] ) OSISAbbrev = BibleOrgSysGlobals.BibleBooksCodes.getOSISAbbreviation( BBB) if not OSISAbbrev: logging.warning( "toOpenSong: Can't write {} OpenSong book because no OSIS code available" .format(BBB)) unhandledBooks.append(BBB) return writerObject.writeLineOpen('b', ('n', bkData.getAssumedBookNames()[0])) haveOpenChapter, startedFlag, gotVP, accumulator = False, False, None, "" C, V = '-1', '-1' # So first/id line starts at -1:0 for processedBibleEntry in bkData._processedLines: # Process internal Bible data lines marker, text, extras = processedBibleEntry.getMarker( ), processedBibleEntry.getCleanText( ), processedBibleEntry.getExtras() #print( marker, repr(text) ) #if text: assert text[0] != ' ' if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS: continue # Just ignore added markers -- not needed here if marker in USFM_PRECHAPTER_MARKERS: if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag: assert C == '-1' or marker == 'rem' or marker.startswith( 'mte') V = str(int(V) + 1) if marker in OFTEN_IGNORED_USFM_HEADER_MARKERS or marker in ( 'ie', ): # Just ignore these lines ignoredMarkers.add(marker) elif marker == 'c': if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) accumulator = '' if haveOpenChapter: writerObject.writeLineClose('c') C, V = text, '0' writerObject.writeLineOpen('c', ('n', text)) haveOpenChapter = True elif marker in ( 'c#', ): # These are the markers that we can safely ignore for this export ignoredMarkers.add(marker) elif marker == 'vp#': # This precedes a v field and has the verse number to be printed gotVP = text # Just remember it for now elif marker == 'v': V = text if gotVP: # this is the verse number to be published text = gotVP gotVP = None startedFlag = True if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) accumulator = '' #print( "Text {!r}".format( text ) ) if not text: logging.warning("createOpenSongXML: Missing text for v") continue verseNumberString = text.replace('<', '').replace( '>', '' ).replace( '"', '' ) # Used below but remove anything that'll cause a big XML problem later elif marker in ('mt1','mt2','mt3','mt4', 'mte1','mte2','mte3','mte4', 'ms1','ms2','ms3','ms4', ) \ or marker in USFM_ALL_INTRODUCTION_MARKERS \ or marker in ('s1','s2','s3','s4', 'r','sr','mr', 'd','sp','cd', 'cl','lit', ): ignoredMarkers.add(marker) elif marker in USFM_BIBLE_PARAGRAPH_MARKERS: if BibleOrgSysGlobals.debugFlag: assert not text and not extras ignoredMarkers.add(marker) elif marker in ( 'b', 'nb', 'ib', ): if BibleOrgSysGlobals.debugFlag: assert not text and not extras ignoredMarkers.add(marker) elif marker in ( 'v~', 'p~', ): if BibleOrgSysGlobals.debugFlag: assert text or extras if not text: # this is an empty (untranslated) verse text = '- - -' # but we'll put in a filler if startedFlag: accumulator += (' ' if accumulator else '') + BibleOrgSysGlobals.makeSafeXML(text) else: if text: logging.warning( "toOpenSong: lost text in {} field in {} {}:{} {!r}". format(marker, BBB, C, V, text)) #if BibleOrgSysGlobals.debugFlag: halt if extras: logging.warning( "toOpenSong: lost extras in {} field in {} {}:{}". format(marker, BBB, C, V)) #if BibleOrgSysGlobals.debugFlag: halt unhandledMarkers.add(marker) if extras and marker not in ( 'v~', 'p~', ) and marker not in ignoredMarkers: logging.critical( "toOpenSong: extras not handled for {} at {} {}:{}".format( marker, BBB, C, V)) if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) if haveOpenChapter: writerObject.writeLineClose('c') writerObject.writeLineClose('b') # end of createOpenSongXML.writeOpenSongBook # Set-up our Bible reference system if 'PublicationCode' not in controlDict or controlDict[ 'PublicationCode'] == 'GENERIC': BOS = BibleObject.genericBOS BRL = BibleObject.genericBRL else: BOS = BibleOrganisationalSystem(controlDict['PublicationCode']) BRL = BibleReferenceList(BOS, BibleObject=None) if BibleOrgSysGlobals.verbosityLevel > 2: print(_(" Exporting to OpenSong format…")) try: osOFn = controlDict['OpenSongOutputFilename'] except KeyError: osOFn = 'Bible.osong' filename = BibleOrgSysGlobals.makeSafeFilename(osOFn) xw = MLWriter(filename, outputFolder) xw.setHumanReadable() xw.start() xw.writeLineOpen('Bible') for BBB, bookData in BibleObject.books.items(): writeOpenSongBook(xw, BBB, bookData) xw.writeLineClose('Bible') xw.close() if ignoredMarkers: logging.info("createOpenSongXML: Ignored markers were {}".format( ignoredMarkers)) if BibleOrgSysGlobals.verbosityLevel > 2: print(" " + _("WARNING: Ignored createOpenSongXML markers were {}" ).format(ignoredMarkers)) if unhandledMarkers: logging.warning("createOpenSongXML: Unhandled markers were {}".format( unhandledMarkers)) if BibleOrgSysGlobals.verbosityLevel > 1: print(" " + _("WARNING: Unhandled toOpenSong markers were {}").format( unhandledMarkers)) if unhandledBooks: logging.warning("createOpenSongXML: Unhandled books were {}".format( unhandledBooks)) if BibleOrgSysGlobals.verbosityLevel > 1: print(" " + _("WARNING: Unhandled createOpenSongXML books were {}" ).format(unhandledBooks)) # Now create a zipped version filepath = os.path.join(outputFolder, filename) if BibleOrgSysGlobals.verbosityLevel > 2: print(" Zipping {} OpenSong file…".format(filename)) zf = zipfile.ZipFile(filepath + '.zip', 'w', compression=zipfile.ZIP_DEFLATED) zf.write(filepath, filename) zf.close() if validationSchema: return xw.validate(validationSchema) if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: print(" createOpenSongXML finished successfully.") return True
def createMySwordModule( self, outputFolder, controlDict ): """ Create a SQLite3 database module for the program MySword. self here is a Bible object with _processedLines """ import tarfile from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS from theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine def writeMSBook( sqlObject, BBB, ourGlobals ): """ Writes a book to the MySword sqlObject file. """ nonlocal lineCount bkData = self.books[BBB] if BBB in self.books else None #print( bkData._processedLines ) verseList = BOS.getNumVersesList( BBB ) nBBB = BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber( BBB ) numC, numV = len(verseList), verseList[0] ourGlobals['line'], ourGlobals['lastLine'] = '', None ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals['pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals['pi7'] = False if bkData: # Write book headings (stuff before chapter 1) ourGlobals['line'] = theWordHandleIntroduction( BBB, bkData, ourGlobals ) # Write the verses C = V = 1 ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None while True: verseData = None if bkData: try: result = bkData.getContextVerseData( (BBB,str(C),str(V),) ) verseData, context = result except KeyError: # Missing verses logging.warning( "BibleWriter.toMySword: missing source verse at {} {}:{}".format( BBB, C, V ) ) # Handle some common versification anomalies if (BBB,C,V) == ('JN3',1,14): # Add text for v15 if it exists try: result15 = bkData.getContextVerseData( ('JN3','1','15',) ) verseData15, context15 = result15 verseData.extend( verseData15 ) except KeyError: pass # just ignore it elif (BBB,C,V) == ('REV',12,17): # Add text for v15 if it exists try: result18 = bkData.getContextVerseData( ('REV','12','18',) ) verseData18, context18 = result18 verseData.extend( verseData18 ) except KeyError: pass # just ignore it composedLine = '' if verseData: composedLine = theWordComposeVerseLine( BBB, C, V, verseData, ourGlobals ) # Stay one line behind (because paragraph indicators get appended to the previous line) if ourGlobals['lastBCV'] is not None \ and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 ourGlobals['lastLine'] = composedLine ourGlobals['lastBCV'] = (nBBB,C,V) V += 1 if V > numV: C += 1 if C > numC: break else: # next chapter only numV = verseList[C-1] V = 1 #assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything # Write the last line of the file if ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 # end of toMySword.writeMSBook # Set-up their Bible reference system BOS = BibleOrganizationalSystem( 'GENERIC-KJV-66-ENG' ) #BRL = BibleReferenceList( BOS, BibleObject=None ) # Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.) if len(self) <= (39+4) and self.containsAnyOT39Books() and not self.containsAnyNT27Books(): testament, startBBB, endBBB = 'OT', 'GEN', 'MAL' booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines elif len(self) <= (27+4) and self.containsAnyNT27Books() and not self.containsAnyOT39Books(): testament, startBBB, endBBB = 'NT', 'MAT', 'REV' booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines else: # assume it's an entire Bible testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV' booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines extension = '.bbl.mybible' if BibleOrgSysGlobals.verbosityLevel > 2: print( _(" Exporting to MySword format…") ) mySettings = {} mySettings['unhandledMarkers'] = set() handledBooks = [] if 'MySwordOutputFilename' in controlDict: filename = controlDict['MySwordOutputFilename'] elif self.sourceFilename: filename = self.sourceFilename elif self.shortName: filename = self.shortName elif self.abbreviation: filename = self.abbreviation elif self.name: filename = self.name else: filename = 'export' if not filename.endswith( extension ): filename += extension # Make sure that we have the right file extension filepath = os.path.join( outputFolder, BibleOrgSysGlobals.makeSafeFilename( filename ) ) if os.path.exists( filepath ): os.remove( filepath ) if BibleOrgSysGlobals.verbosityLevel > 2: print( " " + _("Writing {!r}…").format( filepath ) ) conn = sqlite3.connect( filepath ) cursor = conn.cursor() # First write the settings Details table exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete customCSS = self.getSetting( 'CustomCSS' ) if customCSS: exeStr += ', CustomCSS TEXT' exeStr += ')' cursor.execute( exeStr ) values = [] description = self.getSetting( 'Description' ) if not description: description = self.getSetting( 'description' ) if not description: description = self.name values.append( description ) if self.abbreviation: abbreviation = self.abbreviation else: abbreviation = self.getSetting( 'WorkAbbreviation' ) if not abbreviation: abbreviation = self.name[:3].upper() values.append( abbreviation ) comments = self.getSetting( 'Comments' ) values.append( comments ) version = self.getSetting( 'Version' ) values.append( version ) versionDate = self.getSetting( 'VersionDate' ) values.append( versionDate ) publishDate = self.getSetting( 'PublishDate' ) values.append( publishDate ) rightToLeft = self.getSetting( 'RightToLeft' ) values.append( rightToLeft ) values.append( True if testament=='OT' or testament=='BOTH' else False ) values.append( True if testament=='NT' or testament=='BOTH' else False ) Strong = self.getSetting( 'Strong' ) values.append( Strong if Strong else False ) if customCSS: values.append( customCSS ) exeStr = 'INSERT INTO "Details" VALUES(' + '?,'*(len(values)-1) + '?)' #print( exeStr, values ) cursor.execute( exeStr, values ) #if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values ) #else: # Not debugging #try: cursor.execute( exeStr, values ) #except sqlite3.InterfaceError: #logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) ) # Now create and fill the Bible table cursor.execute( 'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))' ) conn.commit() # save (commit) the changes BBB, lineCount = startBBB, 0 while True: # Write each Bible book in the KJV order writeMSBook( cursor, BBB, mySettings ) conn.commit() # save (commit) the changes handledBooks.append( BBB ) if BBB == endBBB: break BBB = BOS.getNextBookCode( BBB ) conn.commit() # save (commit) the changes cursor.close() if mySettings['unhandledMarkers']: logging.warning( "BibleWriter.toMySword: Unhandled markers were {}".format( mySettings['unhandledMarkers'] ) ) if BibleOrgSysGlobals.verbosityLevel > 1: print( " " + _("WARNING: Unhandled toMySword markers were {}").format( mySettings['unhandledMarkers'] ) ) unhandledBooks = [] for BBB in self.getBookList(): if BBB not in handledBooks: unhandledBooks.append( BBB ) if unhandledBooks: logging.warning( "toMySword: Unhandled books were {}".format( unhandledBooks ) ) if BibleOrgSysGlobals.verbosityLevel > 1: print( " " + _("WARNING: Unhandled toMySword books were {}").format( unhandledBooks ) ) # Now create the gzipped file if BibleOrgSysGlobals.verbosityLevel > 2: print( " Compressing {} MySword file…".format( filename ) ) tar = tarfile.open( filepath+'.gz', 'w:gz' ) tar.add( filepath ) tar.close() if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: print( " BibleWriter.toMySword finished successfully." ) return True
def createMySwordModule(self, outputFolder, controlDict): """ Create a SQLite3 database module for the program MySword. self here is a Bible object with _processedLines """ import tarfile from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS from theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine def writeMSBook(sqlObject, BBB, ourGlobals): """ Writes a book to the MySword sqlObject file. """ nonlocal lineCount bkData = self.books[BBB] if BBB in self.books else None #print( bkData._processedLines ) verseList = BOS.getNumVersesList(BBB) nBBB = BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber(BBB) numC, numV = len(verseList), verseList[0] ourGlobals['line'], ourGlobals['lastLine'] = '', None ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals[ 'pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals[ 'pi7'] = False if bkData: # Write book headings (stuff before chapter 1) ourGlobals['line'] = theWordHandleIntroduction( BBB, bkData, ourGlobals) # Write the verses C = V = 1 ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None while True: verseData = None if bkData: try: result = bkData.getContextVerseData(( BBB, str(C), str(V), )) verseData, context = result except KeyError: # Missing verses logging.warning( "BibleWriter.createMySwordModule: missing source verse at {} {}:{}" .format(BBB, C, V)) # Handle some common versification anomalies if (BBB, C, V) == ('JN3', 1, 14): # Add text for v15 if it exists try: result15 = bkData.getContextVerseData(( 'JN3', '1', '15', )) verseData15, context15 = result15 verseData.extend(verseData15) except KeyError: pass # just ignore it elif (BBB, C, V) == ('REV', 12, 17): # Add text for v15 if it exists try: result18 = bkData.getContextVerseData(( 'REV', '12', '18', )) verseData18, context18 = result18 verseData.extend(verseData18) except KeyError: pass # just ignore it composedLine = '' if verseData: composedLine = theWordComposeVerseLine( BBB, C, V, verseData, ourGlobals) # Stay one line behind (because paragraph indicators get appended to the previous line) if ourGlobals['lastBCV'] is not None \ and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 ourGlobals['lastLine'] = composedLine ourGlobals['lastBCV'] = (nBBB, C, V) V += 1 if V > numV: C += 1 if C > numC: break else: # next chapter only numV = verseList[C - 1] V = 1 #assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything # Write the last line of the file if ourGlobals[ 'lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 # end of createMySwordModule.writeMSBook # Set-up their Bible reference system BOS = BibleOrganisationalSystem('GENERIC-KJV-66-ENG') #BRL = BibleReferenceList( BOS, BibleObject=None ) # Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.) if len(self) <= (39 + 4) and self.containsAnyOT39Books( ) and not self.containsAnyNT27Books(): testament, startBBB, endBBB = 'OT', 'GEN', 'MAL' booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines elif len(self) <= (27 + 4) and self.containsAnyNT27Books( ) and not self.containsAnyOT39Books(): testament, startBBB, endBBB = 'NT', 'MAT', 'REV' booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines else: # assume it's an entire Bible testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV' booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines extension = '.bbl.mybible' if BibleOrgSysGlobals.verbosityLevel > 2: print(_(" Exporting to MySword format…")) mySettings = {} mySettings['unhandledMarkers'] = set() handledBooks = [] if 'MySwordOutputFilename' in controlDict: filename = controlDict['MySwordOutputFilename'] elif self.sourceFilename: filename = self.sourceFilename elif self.shortName: filename = self.shortName elif self.abbreviation: filename = self.abbreviation elif self.name: filename = self.name else: filename = 'export' if not filename.endswith(extension): filename += extension # Make sure that we have the right file extension filepath = os.path.join(outputFolder, BibleOrgSysGlobals.makeSafeFilename(filename)) if os.path.exists(filepath): os.remove(filepath) if BibleOrgSysGlobals.verbosityLevel > 2: print(' createMySwordModule: ' + _("Writing {!r}…").format(filepath)) conn = sqlite3.connect(filepath) cursor = conn.cursor() # First write the settings Details table exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete customCSS = self.getSetting('CustomCSS') if customCSS: exeStr += ', CustomCSS TEXT' exeStr += ')' cursor.execute(exeStr) values = [] description = self.getSetting('Description') if not description: description = self.getSetting('description') if not description: description = self.name values.append(description) if self.abbreviation: abbreviation = self.abbreviation else: abbreviation = self.getSetting('WorkAbbreviation') if not abbreviation: abbreviation = self.name[:3].upper() values.append(abbreviation) comments = self.getSetting('Comments') values.append(comments) version = self.getSetting('Version') values.append(version) versionDate = self.getSetting('VersionDate') values.append(versionDate) publishDate = self.getSetting('PublishDate') values.append(publishDate) rightToLeft = self.getSetting('RightToLeft') values.append(rightToLeft) values.append(True if testament == 'OT' or testament == 'BOTH' else False) values.append(True if testament == 'NT' or testament == 'BOTH' else False) Strong = self.getSetting('Strong') values.append(Strong if Strong else False) if customCSS: values.append(customCSS) exeStr = 'INSERT INTO "Details" VALUES(' + '?,' * (len(values) - 1) + '?)' #print( exeStr, values ) cursor.execute(exeStr, values) #if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values ) #else: # Not debugging #try: cursor.execute( exeStr, values ) #except sqlite3.InterfaceError: #logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) ) # Now create and fill the Bible table cursor.execute( 'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))' ) conn.commit() # save (commit) the changes BBB, lineCount = startBBB, 0 while True: # Write each Bible book in the KJV order writeMSBook(cursor, BBB, mySettings) conn.commit() # save (commit) the changes handledBooks.append(BBB) if BBB == endBBB: break BBB = BOS.getNextBookCode(BBB) conn.commit() # save (commit) the changes cursor.close() if mySettings['unhandledMarkers']: logging.warning( "BibleWriter.createMySwordModule: Unhandled markers were {}". format(mySettings['unhandledMarkers'])) if BibleOrgSysGlobals.verbosityLevel > 1: print(" " + _("WARNING: Unhandled createMySwordModule markers were {}" ).format(mySettings['unhandledMarkers'])) unhandledBooks = [] for BBB in self.getBookList(): if BBB not in handledBooks: unhandledBooks.append(BBB) if unhandledBooks: logging.warning("createMySwordModule: Unhandled books were {}".format( unhandledBooks)) if BibleOrgSysGlobals.verbosityLevel > 1: print(" " + _("WARNING: Unhandled createMySwordModule books were {}" ).format(unhandledBooks)) # Now create the gzipped file if BibleOrgSysGlobals.verbosityLevel > 2: print(" Compressing {} MySword file…".format(filename)) tar = tarfile.open(filepath + '.gz', 'w:gz') tar.add(filepath) tar.close() if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: print(" BibleWriter.createMySwordModule finished successfully.") return True
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion ) if 1: # demo the file checking code -- first with the whole folder and then with only one folder for j,testFolder in enumerate( ("Tests/DataFilesForTests/USFMTest1/", "Tests/DataFilesForTests/USFMTest2/", "Tests/DataFilesForTests/USFMTest3/", "Tests/DataFilesForTests/USFMAllMarkersProject/", "Tests/DataFilesForTests/USFMErrorProject/", "Tests/DataFilesForTests/PTX7Test/", "OutputFiles/BOS_USFM_Export/", "OutputFiles/BOS_USFM_Reexport/", "MadeUpFolder/", ) ): print( "\nUSFM A{} testfolder is: {}".format( j+1, testFolder ) ) result1 = USFMBibleFileCheck( testFolder ) if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAa", result1 ) result2 = USFMBibleFileCheck( testFolder, autoLoad=True ) if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAb", result2 ) result3 = USFMBibleFileCheck( testFolder, autoLoadBooks=True ) if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAc", result3 ) if isinstance( result3, Bible ): if BibleOrgSysGlobals.strictCheckingFlag: result3.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) UsfmBErrors = UsfmB.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: result3.pickle() ##UsfmB.toDrupalBible() result3.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) if 1: # Load and process some of our test versions for j,(name, encoding, testFolder) in enumerate( ( ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest1/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest2/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest3/"), ("WEB+", 'utf-8', "Tests/DataFilesForTests/USFMAllMarkersProject/"), ("UEP", 'utf-8', "Tests/DataFilesForTests/USFMErrorProject/"), ("Exported", 'utf-8', "Tests/BOS_USFM_Export/"), ) ): if os.access( testFolder, os.R_OK ): if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nUSFM B{}/".format( j+1 ) ) UsfmB = USFMBible( testFolder, name, encoding=encoding ) UsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 1: print( "Gen assumed book name:", repr( UsfmB.getAssumedBookName( 'GEN' ) ) ) print( "Gen long TOC book name:", repr( UsfmB.getLongTOCName( 'GEN' ) ) ) print( "Gen short TOC book name:", repr( UsfmB.getShortTOCName( 'GEN' ) ) ) print( "Gen book abbreviation:", repr( UsfmB.getBooknameAbbreviation( 'GEN' ) ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( UsfmB ) if BibleOrgSysGlobals.strictCheckingFlag: UsfmB.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) UsfmBErrors = UsfmB.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: UsfmB.pickle() ##UsfmB.toDrupalBible() UsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) ) if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj ) elif BibleOrgSysGlobals.verbosityLevel > 0: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) ) if 0: # Test a whole folder full of folders of USFM Bibles testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/" def findInfo( somepath ): """ Find out info about the project from the included copyright.htm file """ cFilepath = os.path.join( somepath, "copyright.htm" ) if not os.path.exists( cFilepath ): return with open( cFilepath ) as myFile: # Automatically closes the file when done lastLine, lineCount = None, 0 title, nameDict = None, {} for line in myFile: lineCount += 1 if lineCount==1: if line[0]==chr(65279): #U+FEFF logging.info( "USFMBible.findInfo1: Detected Unicode Byte Order Marker (BOM) in {}".format( "copyright.htm" ) ) line = line[1:] # Remove the UTF-16 Unicode Byte Order Marker (BOM) elif line[:3] == '': # 0xEF,0xBB,0xBF logging.info( "USFMBible.findInfo2: Detected Unicode Byte Order Marker (BOM) in {}".format( "copyright.htm" ) ) line = line[3:] # Remove the UTF-8 Unicode Byte Order Marker (BOM) if line[-1]=='\n': line = line[:-1] # Removing trailing newline character if not line: continue # Just discard blank lines lastLine = line if line.startswith("<title>"): title = line.replace("<title>","").replace("</title>","").strip() if line.startswith('<option value="'): adjLine = line.replace('<option value="','').replace('</option>','') USFM_BBB, name = adjLine[:3], adjLine[11:] BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromUSFM( USFM_BBB ) #print( USFM_BBB, BBB, name ) nameDict[BBB] = name return title, nameDict # end of findInfo count = totalBooks = 0 if os.access( testBaseFolder, os.R_OK ): # check that we can read the test data for something in sorted( os.listdir( testBaseFolder ) ): somepath = os.path.join( testBaseFolder, something ) if os.path.isfile( somepath ): print( "Ignoring file {!r} in {!r}".format( something, testBaseFolder ) ) elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a USFM (partial) Bible #if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules count += 1 title = None findInfoResult = findInfo( somepath ) if findInfoResult: title, bookNameDict = findInfoResult if title is None: title = something[:-5] if something.endswith("_usfm") else something name, encoding, testFolder = title, 'utf-8', somepath if os.access( testFolder, os.R_OK ): if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nUSFM C{}/".format( count ) ) UsfmB = USFMBible( testFolder, name, encoding=encoding ) UsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 0: print( UsfmB ) if BibleOrgSysGlobals.strictCheckingFlag: UsfmB.check() UsfmBErrors = UsfmB.getErrors() #print( UsfmBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: UsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) ) if count: print( "\n{} total USFM (partial) Bibles processed.".format( count ) ) if totalBooks: print( "{} total books ({} average per folder)".format( totalBooks, round(totalBooks/count) ) ) elif BibleOrgSysGlobals.verbosityLevel > 0: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testBaseFolder ) )
def demo(): """ Demonstrate reading and checking some Bible databases. """ if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion) if 1: # demo the file checking code -- first with the whole folder and then with only one folder for j, testFolder in enumerate(( "Tests/DataFilesForTests/USFMTest1/", "Tests/DataFilesForTests/USFMTest2/", "Tests/DataFilesForTests/USFMTest3/", "Tests/DataFilesForTests/USFMAllMarkersProject/", "Tests/DataFilesForTests/USFMErrorProject/", "Tests/DataFilesForTests/PTX7Test/", "OutputFiles/BOS_USFM_Export/", "OutputFiles/BOS_USFM_Reexport/", "MadeUpFolder/", )): print("\nUSFM A{} testfolder is: {}".format(j + 1, testFolder)) result1 = USFMBibleFileCheck(testFolder) if BibleOrgSysGlobals.verbosityLevel > 1: print("USFM TestAa", result1) result2 = USFMBibleFileCheck(testFolder, autoLoad=True) if BibleOrgSysGlobals.verbosityLevel > 1: print("USFM TestAb", result2) result3 = USFMBibleFileCheck(testFolder, autoLoadBooks=True) if BibleOrgSysGlobals.verbosityLevel > 1: print("USFM TestAc", result3) if isinstance(result3, Bible): if BibleOrgSysGlobals.strictCheckingFlag: result3.check() #print( result3.books['GEN']._processedLines[0:40] ) UsfmBErrors = result3.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: result3.pickle() ##result3.toDrupalBible() result3.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) if 1: # Load and process some of our test versions for j, (name, encoding, testFolder) in enumerate(( ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest1/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest2/"), ("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest3/"), ("WEB+", 'utf-8', "Tests/DataFilesForTests/USFMAllMarkersProject/"), ("UEP", 'utf-8', "Tests/DataFilesForTests/USFMErrorProject/"), ("Exported", 'utf-8', "Tests/BOS_USFM_Export/"), )): if os.access(testFolder, os.R_OK): if BibleOrgSysGlobals.verbosityLevel > 0: print("\nUSFM B{}/".format(j + 1)) UsfmB = USFMBible(testFolder, name, encoding=encoding) UsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 1: print("Gen assumed book name:", repr(UsfmB.getAssumedBookName('GEN'))) print("Gen long TOC book name:", repr(UsfmB.getLongTOCName('GEN'))) print("Gen short TOC book name:", repr(UsfmB.getShortTOCName('GEN'))) print("Gen book abbreviation:", repr(UsfmB.getBooknameAbbreviation('GEN'))) if BibleOrgSysGlobals.verbosityLevel > 0: print(UsfmB) if BibleOrgSysGlobals.strictCheckingFlag: UsfmB.check() #print( UsfmB.books['GEN']._processedLines[0:40] ) UsfmBErrors = UsfmB.getErrors() # print( UBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: UsfmB.pickle() ##UsfmB.toDrupalBible() UsfmB.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join("OutputFiles/", "BOS_Bible_Object_Pickle/")) if BibleOrgSysGlobals.verbosityLevel > 0: print("newObj is", newObj) elif BibleOrgSysGlobals.verbosityLevel > 0: print( "\nSorry, test folder {!r} is not readable on this computer." .format(testFolder)) if 0: # Test a whole folder full of folders of USFM Bibles testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/" def findInfo(somepath): """ Find out info about the project from the included copyright.htm file """ cFilepath = os.path.join(somepath, "copyright.htm") if not os.path.exists(cFilepath): return with open(cFilepath ) as myFile: # Automatically closes the file when done lastLine, lineCount = None, 0 title, nameDict = None, {} for line in myFile: lineCount += 1 if lineCount == 1: if line[0] == chr(65279): #U+FEFF logging.info( "USFMBible.findInfo1: Detected Unicode Byte Order Marker (BOM) in {}" .format("copyright.htm")) line = line[ 1:] # Remove the UTF-16 Unicode Byte Order Marker (BOM) elif line[:3] == '': # 0xEF,0xBB,0xBF logging.info( "USFMBible.findInfo2: Detected Unicode Byte Order Marker (BOM) in {}" .format("copyright.htm")) line = line[ 3:] # Remove the UTF-8 Unicode Byte Order Marker (BOM) if line[-1] == '\n': line = line[:-1] # Removing trailing newline character if not line: continue # Just discard blank lines lastLine = line if line.startswith("<title>"): title = line.replace("<title>", "").replace("</title>", "").strip() if line.startswith('<option value="'): adjLine = line.replace('<option value="', '').replace('</option>', '') USFM_BBB, name = adjLine[:3], adjLine[11:] BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromUSFM( USFM_BBB) #print( USFM_BBB, BBB, name ) nameDict[BBB] = name return title, nameDict # end of findInfo count = totalBooks = 0 if os.access(testBaseFolder, os.R_OK): # check that we can read the test data for something in sorted(os.listdir(testBaseFolder)): somepath = os.path.join(testBaseFolder, something) if os.path.isfile(somepath): print("Ignoring file {!r} in {!r}".format( something, testBaseFolder)) elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a USFM (partial) Bible #if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules count += 1 title = None findInfoResult = findInfo(somepath) if findInfoResult: title, bookNameDict = findInfoResult if title is None: title = something[:-5] if something.endswith( "_usfm") else something name, encoding, testFolder = title, 'utf-8', somepath if os.access(testFolder, os.R_OK): if BibleOrgSysGlobals.verbosityLevel > 0: print("\nUSFM C{}/".format(count)) UsfmB = USFMBible(testFolder, name, encoding=encoding) UsfmB.load() if BibleOrgSysGlobals.verbosityLevel > 0: print(UsfmB) if BibleOrgSysGlobals.strictCheckingFlag: UsfmB.check() UsfmBErrors = UsfmB.getErrors() #print( UsfmBErrors ) if BibleOrgSysGlobals.commandLineArguments.export: UsfmB.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) else: print( "\nSorry, test folder {!r} is not readable on this computer." .format(testFolder)) if count: print("\n{} total USFM (partial) Bibles processed.".format( count)) if totalBooks: print("{} total books ({} average per folder)".format( totalBooks, round(totalBooks / count))) elif BibleOrgSysGlobals.verbosityLevel > 0: print( "\nSorry, test folder {!r} is not readable on this computer.". format(testBaseFolder))
def __init__(self, parameterOne, resourcesObject=None, downloadAllBooks=False): """ Create the Door43 cataloged Bible object. parameterOne can be: a catalog dictionary entry (and second parameter must be None) or an index into the BibleList in the resourcesObject passed as the second parameter """ if BibleOrgSysGlobals.debugFlag and debuggingThisModule: print( f"DCSBible.__init__( {parameterOne}, {resourcesObject}, {downloadAllBooks} )…" ) if isinstance(parameterOne, dict): assert resourcesObject is None resourceDict = parameterOne else: assert isinstance(parameterOne, int) assert resourcesObject # why ??? and isinstance( resourcesObject, Door43CatalogResources ) resourceDict = resourcesObject.getBibleResourceDict(parameterOne) assert resourceDict and isinstance(resourceDict, dict) #print( 'resourceDict', resourceDict ) #print( 'resourceDict', resourceDict.keys() ) self.baseURL = resourceDict['html_url'] #print( 'self.baseURL', self.baseURL ) adjustedRepoName = resourceDict['full_name'].replace('/', '--') #print( 'adjustedRepoName', adjustedRepoName ) desiredFolderName = BibleOrgSysGlobals.makeSafeFilename( adjustedRepoName) unzippedFolderPath = os.path.join( BibleOrgSysGlobals.DOWNLOADED_RESOURCES_FOLDER, 'Door43ContentServiceOnline/', f"{adjustedRepoName}/") if downloadAllBooks: # See if files already exist and are current (so don't download again) alreadyDownloadedFlag = False if os.path.isdir(unzippedFolderPath): #print( f"Issued: {resourceDict['issued']}" ) updatedDatetime = datetime.strptime(resourceDict['updated_at'], '%Y-%m-%dT%H:%M:%SZ') #print( f"updatedDatetime: {updatedDatetime}" ) #print( f"folder: {os.stat(unzippedFolderPath).st_mtime}" ) folderModifiedDatetime = datetime.fromtimestamp( os.stat(unzippedFolderPath).st_mtime) #print( f"folderModifiedDatetime: {folderModifiedDatetime}" ) alreadyDownloadedFlag = folderModifiedDatetime > updatedDatetime #print( f"alreadyDownloadedFlag: {alreadyDownloadedFlag}" ) if alreadyDownloadedFlag: if BibleOrgSysGlobals.verbosityLevel > 1: print( "Skipping download because folder '{}' already exists." .format(unzippedFolderPath)) else: # Download the zip file (containing all the USFM files, README.md, LICENSE.md, manifest.yaml, etc.) # TODO: Change to .tar.gz instead of zip zipURL = self.baseURL + '/archive/master.zip' # '/archive/master.tar.gz' if BibleOrgSysGlobals.verbosityLevel > 1: print("Downloading entire repo from '{}'…".format(zipURL)) try: HTTPResponseObject = urllib.request.urlopen(zipURL) except urllib.error.URLError as err: #errorClass, exceptionInstance, traceback = sys.exc_info() #print( '{!r} {!r} {!r}'.format( errorClass, exceptionInstance, traceback ) ) logging.critical("DCS URLError '{}' from {}".format( err, zipURL)) return # print( " HTTPResponseObject", HTTPResponseObject ) contentType = HTTPResponseObject.info().get('content-type') if BibleOrgSysGlobals.debugFlag and debuggingThisModule: print(" contentType", repr(contentType)) if contentType == 'application/octet-stream': try: os.makedirs(unzippedFolderPath) except FileExistsError: pass downloadedData = HTTPResponseObject.read() if BibleOrgSysGlobals.verbosityLevel > 0: print( f" Downloaded {len(downloadedData):,} bytes from '{zipURL}'" ) # Bug in Python up to 3.7 makes this not work for large aligned Bibles (3+ MB) # myTempFile = tempfile.SpooledTemporaryFile() myTempFile = tempfile.TemporaryFile() myTempFile.write(downloadedData) with zipfile.ZipFile(myTempFile) as myzip: # NOTE: Could be a security risk here myzip.extractall(unzippedFolderPath) myTempFile.close() # Automatically deletes the file else: print(" contentType", repr(contentType)) halt # unknown content type self.downloadedAllBooks = True # There's probably a folder inside this folder folders = os.listdir(unzippedFolderPath) #print( 'folders', folders ) assert len( folders ) == 1 # else maybe a previous download failed -- just manually delete the folder desiredFolderName = folders[0] + '/' #print( 'desiredFolderName', desiredFolderName ) USFMBible.__init__(self, os.path.join(unzippedFolderPath, desiredFolderName), givenName=resourceDict['name']) else: self.downloadedAllBooks = False self.attemptedDownload = {} try: os.makedirs(unzippedFolderPath) except FileExistsError: pass USFMBible.__init__(self, unzippedFolderPath, givenName=resourceDict['name']) self.objectNameString = 'DCS USFM Bible object'
def createEasyWorshipBible( BibleObject, outputFolder=None ): """ Write the pseudo USFM out into the compressed EasyWorship format. Since we don't have a specification for the format, and since we don't know the meaning of all the binary pieces of the file, we can't be certain yet that this output will actually work. :-( """ from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS import zipfile # It seems 7-9 give the correct two header bytes ZLIB_COMPRESSION_LEVEL = 9 # -1=default(=6), 0=none, 1=fastest...9=highest compression level if BibleOrgSysGlobals.verbosityLevel > 1: print( "Running createEasyWorshipBible…" ) if BibleOrgSysGlobals.debugFlag: assert BibleObject.books if not BibleObject.doneSetupGeneric: BibleObject.__setupWriter() if not outputFolder: outputFolder = 'OutputFiles/BOS_EasyWorshipBible_Export/' if not os.access( outputFolder, os.F_OK ): os.makedirs( outputFolder ) # Make the empty folder if there wasn't already one there # Set-up their Bible reference system BOS = BibleOrganizationalSystem( 'GENERIC-KJV-66-ENG' ) ignoredMarkers = set() # Before we write the file, let's compress all our books # Books are written as C:V verseText with double-spaced lines compressedDictionary = {} for BBB,bookObject in BibleObject.books.items(): if BBB in ('FRT','INT','BAK','OTH','GLS','XXA','XXB','XXC','XXD','XXE','XXF','XXG',): continue # Ignore these books pseudoESFMData = bookObject._processedLines textBuffer = '' vBridgeStartInt = vBridgeEndInt = None # For printing missing (bridged) verse numbers for entry in pseudoESFMData: marker, text = entry.getMarker(), entry.getCleanText() #print( BBB, marker, text ) if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS: continue # Just ignore added markers -- not needed here elif marker == 'c': C = int( text ) # Just so we get an error if we have something different V = lastVWritten = '0' elif marker == 'v': #V = text.replace( '–', '-' ).replace( '—', '-' ) # Replace endash, emdash with hyphen V = text for bridgeChar in ('-', '–', '—'): # hyphen, endash, emdash ix = V.find( bridgeChar ) if ix != -1: if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.verbosityLevel>2: print( "createEasyWorshipBible: preparing for verse bridge in {} at {} {}:{}" \ .format( BibleObject.abbreviation, BBB, C, V ) ) # Remove verse bridges vStart = V[:ix].replace( 'a', '' ).replace( 'b', '' ).replace( 'c', '' ) vEnd = V[ix+1:].replace( 'a', '' ).replace( 'b', '' ).replace( 'c', '' ) #print( BBB, repr(vStart), repr(vEnd) ) try: vBridgeStartInt, vBridgeEndInt = int( vStart ), int( vEnd ) except ValueError: print( "createEasyWorshipBible: bridge doesn't seem to be integers in {} {}:{!r}".format( BBB, C, V ) ) vBridgeStartInt = vBridgeEndInt = None # One of them isn't an integer #print( ' ', BBB, repr(vBridgeStartInt), repr(vBridgeEndInt) ) VBridgedText = V V = vStart break elif marker == 'v~': try: if int(V) <= int(lastVWritten): # TODO: Not sure what level the following should be? info/warning/error/critical ???? logging.warning( 'createEasyWorshipBible: Maybe duplicating {} {}:{} after {} with {}'.format( BBB, C, V, lastVWritten, text ) ) #continue except ValueError: pass # had a verse bridge if vBridgeStartInt and vBridgeEndInt: # We had a verse bridge if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.verbosityLevel>2: print( "createEasyWorshipBible: handling verse bridge in {} at {} {}:{}-{}" \ .format( BibleObject.abbreviation, BBB, C, vBridgeStartInt, vBridgeEndInt ) ) if 1: # new code -- copies the bridged text to all verses for vNum in range( vBridgeStartInt, vBridgeEndInt+1 ): # Fill in missing verse numbers textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} ({}) {}'.format( C, vNum, VBridgedText, text ) else: # old code textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} ({}) {}'.format( C, vBridgeStartInt, vBridgeEndInt, text ) for vNum in range( vBridgeStartInt+1, vBridgeEndInt+1 ): # Fill in missing verse numbers textBuffer += '\r\n\r\n{}:{} (-)'.format( C, vNum ) lastVWritten = str( vBridgeEndInt ) vBridgeStartInt = vBridgeEndInt = None else: textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} {}'.format( C, V, text ) lastVWritten = V elif marker == 'p~': if BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag: assert textBuffer # This is a continued part of the verse -- failed with this bad source USFM: # \c 1 \v 1 \p These events happened... textBuffer += ' {}'.format( text ) # continuation of the same verse else: ignoredMarkers.add( marker ) #print( BBB, textBuffer ) textBuffer = textBuffer \ .replace( '“', '"' ).replace( '”', '"' ) \ .replace( "‘", "'" ).replace( "’", "'" ) \ .replace( '–', '--' ).replace( '—', '--' ) bookBytes = zlib.compress( textBuffer.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL ) #print( BBB, hexlify(bookBytes[:20]), bookBytes ) assert bookBytes[0]==0x78 and bookBytes[1]==0xda # Zlib compression header appendage = b'QK\x03\x04' + struct.pack( '<I', len(textBuffer) ) + b'\x08\x00' #print( "appendage", len(appendage), hexlify(appendage), appendage ) assert len(appendage) == 10 compressedDictionary[BBB] = bookBytes + appendage # Work out the "compressed" (osfuscated) module name #name = BibleObject.getAName() ##print( 'sn', repr(BibleObject.shortName) ) #if len(name)>18: #if BibleObject.shortName: name = shortName #elif name.endswith( ' Version' ): name = name[:-8] #name = name.replace( ' ', '' ) #if not name.startswith( 'ezFree' ): name = 'ezFree' + name name = 'ezFree' + ( BibleObject.abbreviation if BibleObject.abbreviation else 'UNK' ) if len(name)>16: name = name[:16] # Shorten encodedNameBytes = zlib.compress( name.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL ) if BibleOrgSysGlobals.debugFlag: print( 'Name {!r} went from {} to {} bytes'.format( name, len(name), len(encodedNameBytes) ) ) assert encodedNameBytes[0]==0x78 and encodedNameBytes[1]==0xda # Zlib compression header assert len(encodedNameBytes) <= 26 filename = '{}{}'.format( BibleObject.abbreviation, FILENAME_ENDING ).lower() filepath = os.path.join( outputFolder, BibleOrgSysGlobals.makeSafeFilename( filename ) ) if BibleOrgSysGlobals.verbosityLevel > 2: print( ' createEasyWorshipBible: ' + _("Writing {!r}…").format( filepath ) ) bookAddress = startingBookAddress = 14872 + len(name) + 18 + 4 # Name is something like ezFreeXXX vBridgeStartInt = vBridgeEndInt = None # For printing missing (bridged) verse numbers with open( filepath, 'wb' ) as myFile: assert myFile.tell() == 0 # Write the header info to binary file myFile.write( b'EasyWorship Bible Text\x1a\x02<\x00\x00\x00\xe0\x00\x00\x00' ) assert myFile.tell() == 32 nameBytes = ( BibleObject.getAName() ).encode( 'utf8' ) myFile.write( nameBytes + b'\x00' * (56 - len(nameBytes)) ) assert myFile.tell() == 88 # 32 + 56 # Write the numChapters,numVerses info along with the file position and length for BBB in BOS.getBookList(): #bookName = BibleObject.getAssumedBookName( BBB ) try: bookName = BibleObject.books[BBB].shortTOCName except (KeyError,AttributeError): bookName = None # KeyError if no BBB, AttributeError if no shortTOCName #print( len(bookName) if bookName else '', bookName ) assert bookName is None or len(bookName) <= 51 if bookName: bookNameBytes = bookName.encode( 'utf8' ) else: bookNameBytes = b'' # Not compulsory -- will default to English myFile.write( bookNameBytes + b'\x00' * (51 - len(bookNameBytes)) ) numVersesList = BOS.getNumVersesList( BBB ) numChapters = len( numVersesList ) myFile.write( struct.pack( 'B', numChapters ) ) for verseCount in numVersesList: myFile.write( struct.pack( 'B', verseCount ) ) myFile.write( b'\x00' * (157 - numChapters - 1) ) try: bookBytes = compressedDictionary[BBB] # if it exists except KeyError: # Fill in missing books missingString = "1:1 Book not available\r\n\r\n" bookBytes = zlib.compress( missingString.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL ) assert bookBytes[0]==0x78 and bookBytes[1]==0xda # Zlib compression header appendage = b'QK\x03\x04' + struct.pack( '<I', len(missingString) ) + b'\x08\x00' assert len(appendage) == 10 bookBytes += appendage compressedDictionary[BBB] = bookBytes myFile.write( struct.pack( '<Q', bookAddress ) ) myFile.write( struct.pack( '<Q', len(bookBytes) ) ) bookAddress += len(bookBytes) assert myFile.tell() == 14872 # 32 + 56 + 224*66 # Write the "compressed" (osfuscated) module name myFile.write( struct.pack( '<I', len(name) + 18 ) ) assert myFile.tell() == 14876 # 32 + 56 + 224*66 + 4 myFile.write( encodedNameBytes ) appendage = b'QK\x03\x04' + struct.pack( 'B', len(name) ) + b'\x00' #print( "appendage", len(appendage), hexlify(appendage), appendage ) assert len(appendage) == 6 myFile.write( appendage ) remainderCount = 18 + len(name) - len(encodedNameBytes) - 4 - len(appendage) #print( "remainderCount", remainderCount ) assert remainderCount == 0 #myFile.write( b'\x00' * remainderCount ) myFile.write( b'\x00\x00\x08\x00' ) # Not sure what this means #if debuggingThisModule or BibleOrgSysGlobals.debugFlag: #print( "At", myFile.tell(), 'want', startingBookAddress ) assert myFile.tell() == startingBookAddress # Write the book info to the binary files for BBB in BOS.getBookList(): if BBB in compressedDictionary: myFile.write( compressedDictionary[BBB] ) # Write zlib output elif BibleOrgSysGlobals.verbosityLevel > 2: print( ' Book {} is not available for EasyWorship export'.format( BBB ) ) # Write the end of file stuff myFile.write( b'\x18:\x00\x00\x00\x00\x00\x00ezwBible' ) if ignoredMarkers: logging.info( "createEasyWorshipBible: Ignored markers were {}".format( ignoredMarkers ) ) if BibleOrgSysGlobals.verbosityLevel > 2: print( " " + _("WARNING: Ignored createEasyWorshipBible markers were {}").format( ignoredMarkers ) ) # Now create a zipped version filepath = os.path.join( outputFolder, filename ) if BibleOrgSysGlobals.verbosityLevel > 2: print( " Zipping {} EWB file…".format( filename ) ) zf = zipfile.ZipFile( filepath+'.zip', 'w', compression=zipfile.ZIP_DEFLATED ) zf.write( filepath, filename ) zf.close() if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: print( " BibleWriter.createEasyWorshipBible finished successfully." ) return True