def getChangeLogFilepath( loggingFolder, projectName ):
"""
"""
return os.path.join( loggingFolder, \
BibleOrgSysGlobals.makeSafeFilename( projectName.replace(' ','_') + '_ChangeLog.txt' ) )
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion )
if 1: # Load and process some of our test versions
count = 0
for name, abbreviation, testFolder in ( # name, abbreviation, folder
("Open English Translation—Literal Version", "OET-LV", "../../../../../Data/Work/Matigsalug/Bible/OET-LV/",),
#("Matigsalug", "MBTV", "../../../../../Data/Work/Matigsalug/Bible/MBTV/",),
#("ESFM Test 1", "OET-LV", "Tests/DataFilesForTests/ESFMTest1/"),
#("ESFM Test 2", "OET-RV", "Tests/DataFilesForTests/ESFMTest2/"),
#("All Markers Project", "WEB+", "Tests/DataFilesForTests/USFMAllMarkersProject/"),
#("USFM Error Project", "UEP", "Tests/DataFilesForTests/USFMErrorProject/"),
#("BOS Exported Files", "Exported", "Tests/BOS_USFM_Export/"),
):
count += 1
if os.access( testFolder, os.R_OK ):
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nESFM A{}/".format( count ) )
EsfmB = ESFMBible( testFolder, name, abbreviation )
EsfmB.load()
print( "Gen assumed book name:", repr( EsfmB.getAssumedBookName( 'GEN' ) ) )
print( "Gen long TOC book name:", repr( EsfmB.getLongTOCName( 'GEN' ) ) )
print( "Gen short TOC book name:", repr( EsfmB.getShortTOCName( 'GEN' ) ) )
print( "Gen book abbreviation:", repr( EsfmB.getBooknameAbbreviation( 'GEN' ) ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( EsfmB )
if BibleOrgSysGlobals.strictCheckingFlag:
EsfmB.check()
#print( EsfmB.books['GEN']._processedLines[0:40] )
EsfmBErrors = EsfmB.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineOptions.export:
##EsfmB.toDrupalBible()
EsfmB.doAllExports( wantPhotoBible=False, wantODFs=True, wantPDFs=True )
newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(abbreviation) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj )
else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
if 0: # Test a whole folder full of folders of ESFM Bibles
testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/"
def findInfo( somepath ):
""" Find out info about the project from the included copyright.htm file """
cFilepath = os.path.join( somepath, "copyright.htm" )
if not os.path.exists( cFilepath ): return
with open( cFilepath, encoding='utf-8' ) as myFile: # Automatically closes the file when done
lastLine, lineCount = None, 0
title, nameDict = None, {}
for line in myFile:
lineCount += 1
if lineCount==1 and line and line[0]==chr(65279): #U+FEFF
logging.info( "ESFMBible: Detected UTF-16 Byte Order Marker in copyright.htm file" )
line = line[1:] # Remove the UTF-8 Byte Order Marker
if line[-1]=='\n': line = line[:-1] # Removing trailing newline character
if not line: continue # Just discard blank lines
lastLine = line
if line.startswith("<title>"): title = line.replace("<title>","").replace("</title>","").strip()
if line.startswith('<option value="'):
adjLine = line.replace('<option value="','').replace('</option>','')
ESFM_BBB, name = adjLine[:3], adjLine[11:]
BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromESFM( ESFM_BBB )
#print( ESFM_BBB, BBB, name )
nameDict[BBB] = name
return title, nameDict
# end of findInfo
count = totalBooks = 0
if os.access( testBaseFolder, os.R_OK ): # check that we can read the test data
for something in sorted( os.listdir( testBaseFolder ) ):
somepath = os.path.join( testBaseFolder, something )
if os.path.isfile( somepath ): print( "Ignoring file {!r} in {!r}".format( something, testBaseFolder ) )
elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a ESFM (partial) Bible
#if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules
count += 1
title = None
findInfoResult = findInfo( somepath )
if findInfoResult: title, bookNameDict = findInfoResult
if title is None: title = something[:-5] if something.endswith("_usfm") else something
name, testFolder = title, somepath
if os.access( testFolder, os.R_OK ):
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nESFM B{}/".format( count ) )
EsfmB = ESFMBible( testFolder, name )
EsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 0: print( EsfmB )
if BibleOrgSysGlobals.strictCheckingFlag:
EsfmB.check()
EsfmBErrors = EsfmB.getErrors()
#print( EsfmBErrors )
if BibleOrgSysGlobals.commandLineOptions.export: EsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
if count: print( "\n{} total ESFM (partial) Bibles processed.".format( count ) )
if totalBooks: print( "{} total books ({} average per folder)".format( totalBooks, round(totalBooks/count) ) )
else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testBaseFolder ) )
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion)
if 1: # Load and process some of our test versions
count = 0
for name, abbreviation, testFolder in ( # name, abbreviation, folder
(
"Open English Translation—Literal Version",
"OET-LV",
"../../../../../Data/Work/Matigsalug/Bible/OET-LV/",
),
#("Matigsalug", "MBTV", "../../../../../Data/Work/Matigsalug/Bible/MBTV/",),
#("ESFM Test 1", "OET-LV", "Tests/DataFilesForTests/ESFMTest1/"),
#("ESFM Test 2", "OET-RV", "Tests/DataFilesForTests/ESFMTest2/"),
#("All Markers Project", "WEB+", "Tests/DataFilesForTests/USFMAllMarkersProject/"),
#("USFM Error Project", "UEP", "Tests/DataFilesForTests/USFMErrorProject/"),
#("BOS Exported Files", "Exported", "Tests/BOS_USFM_Export/"),
):
count += 1
if os.access(testFolder, os.R_OK):
if BibleOrgSysGlobals.verbosityLevel > 0:
print("\nESFM A{}/".format(count))
EsfmB = ESFMBible(testFolder, name, abbreviation)
EsfmB.load()
print("Gen assumed book name:",
repr(EsfmB.getAssumedBookName('GEN')))
print("Gen long TOC book name:",
repr(EsfmB.getLongTOCName('GEN')))
print("Gen short TOC book name:",
repr(EsfmB.getShortTOCName('GEN')))
print("Gen book abbreviation:",
repr(EsfmB.getBooknameAbbreviation('GEN')))
if BibleOrgSysGlobals.verbosityLevel > 0: print(EsfmB)
if BibleOrgSysGlobals.strictCheckingFlag:
EsfmB.check()
#print( EsfmB.books['GEN']._processedLines[0:40] )
EsfmBErrors = EsfmB.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineOptions.export:
##EsfmB.toDrupalBible()
EsfmB.doAllExports(wantPhotoBible=False,
wantODFs=True,
wantPDFs=True)
newObj = BibleOrgSysGlobals.unpickleObject(
BibleOrgSysGlobals.makeSafeFilename(abbreviation) +
'.pickle',
os.path.join("OutputFiles/",
"BOS_Bible_Object_Pickle/"))
if BibleOrgSysGlobals.verbosityLevel > 0:
print("newObj is", newObj)
else:
print(
"\nSorry, test folder {!r} is not readable on this computer."
.format(testFolder))
if 0: # Test a whole folder full of folders of ESFM Bibles
testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/"
def findInfo(somepath):
""" Find out info about the project from the included copyright.htm file """
cFilepath = os.path.join(somepath, "copyright.htm")
if not os.path.exists(cFilepath): return
with open(cFilepath, encoding='utf-8'
) as myFile: # Automatically closes the file when done
lastLine, lineCount = None, 0
title, nameDict = None, {}
for line in myFile:
lineCount += 1
if lineCount == 1 and line and line[0] == chr(
65279): #U+FEFF
logging.info(
"ESFMBible: Detected UTF-16 Byte Order Marker in copyright.htm file"
)
line = line[1:] # Remove the UTF-8 Byte Order Marker
if line[-1] == '\n':
line = line[:-1] # Removing trailing newline character
if not line: continue # Just discard blank lines
lastLine = line
if line.startswith("<title>"):
title = line.replace("<title>",
"").replace("</title>",
"").strip()
if line.startswith('<option value="'):
adjLine = line.replace('<option value="',
'').replace('</option>', '')
ESFM_BBB, name = adjLine[:3], adjLine[11:]
BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromESFM(
ESFM_BBB)
#print( ESFM_BBB, BBB, name )
nameDict[BBB] = name
return title, nameDict
# end of findInfo
count = totalBooks = 0
if os.access(testBaseFolder,
os.R_OK): # check that we can read the test data
for something in sorted(os.listdir(testBaseFolder)):
somepath = os.path.join(testBaseFolder, something)
if os.path.isfile(somepath):
print("Ignoring file {!r} in {!r}".format(
something, testBaseFolder))
elif os.path.isdir(
somepath
): # Let's assume that it's a folder containing a ESFM (partial) Bible
#if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules
count += 1
title = None
findInfoResult = findInfo(somepath)
if findInfoResult: title, bookNameDict = findInfoResult
if title is None:
title = something[:-5] if something.endswith(
"_usfm") else something
name, testFolder = title, somepath
if os.access(testFolder, os.R_OK):
if BibleOrgSysGlobals.verbosityLevel > 0:
print("\nESFM B{}/".format(count))
EsfmB = ESFMBible(testFolder, name)
EsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 0: print(EsfmB)
if BibleOrgSysGlobals.strictCheckingFlag:
EsfmB.check()
EsfmBErrors = EsfmB.getErrors()
#print( EsfmBErrors )
if BibleOrgSysGlobals.commandLineOptions.export:
EsfmB.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
else:
print(
"\nSorry, test folder {!r} is not readable on this computer."
.format(testFolder))
if count:
print("\n{} total ESFM (partial) Bibles processed.".format(
count))
if totalBooks:
print("{} total books ({} average per folder)".format(
totalBooks, round(totalBooks / count)))
else:
print(
"\nSorry, test folder {!r} is not readable on this computer.".
format(testBaseFolder))
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion )
#testFolder = "Tests/DataFilesForTests/BCVTest1/"
testFolder = "OutputFiles/BOS_BCV_Export/"
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA1" )
result1 = BCVBibleFileCheck( testFolder )
if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA1", result1 )
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA2" )
result2 = BCVBibleFileCheck( testFolder, autoLoad=True ) # But doesn't preload books
if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA2", result2 )
#result2.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result2.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result2.getErrors()
# print( bibleErrors )
#if BibleOrgSysGlobals.commandLineArguments.export:
###result2.toDrupalBible()
#result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV TestA3" )
result3 = BCVBibleFileCheck( testFolder, autoLoad=True, autoLoadBooks=True )
if BibleOrgSysGlobals.verbosityLevel > 1: print( "BCV TestA3", result3 )
#result3.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result3.getErrors()
# print( bibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##result3.toDrupalBible()
result3.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
if 0: # all discovered modules in the test folder
foundFolders, foundFiles = [], []
for something in os.listdir( testFolder ):
somepath = os.path.join( testFolder, something )
if os.path.isdir( somepath ): foundFolders.append( something )
elif os.path.isfile( somepath ): foundFiles.append( something )
if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work
if BibleOrgSysGlobals.verbosityLevel > 1: print( "\nTrying all {} discovered modules…".format( len(foundFolders) ) )
parameters = [folderName for folderName in sorted(foundFolders)]
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool( processes=BibleOrgSysGlobals.maxProcesses ) as pool: # start worker processes
results = pool.map( testBCV, parameters ) # have the pool do our loads
assert len(results) == len(parameters) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate( sorted( foundFolders ) ):
if BibleOrgSysGlobals.verbosityLevel > 1: print( "\nBCV D{}/ Trying {}".format( j+1, someFolder ) )
#myTestFolder = os.path.join( testFolder, someFolder+'/' )
testBCV( someFolder )
if 0: # Load and process some of our test versions
count = 0
for name, encoding, testFolder in (
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest1/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest2/"),
("Exported", 'utf-8', "Tests/BOS_BCV_Export/"),
):
count += 1
if os.access( testFolder, os.R_OK ):
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nBCV A{}/".format( count ) )
bcvB = BCVBible( testFolder, name, encoding=encoding )
bcvB.load()
if BibleOrgSysGlobals.verbosityLevel > 1:
print( "Gen assumed book name:", repr( bcvB.getAssumedBookName( 'GEN' ) ) )
print( "Gen long TOC book name:", repr( bcvB.getLongTOCName( 'GEN' ) ) )
print( "Gen short TOC book name:", repr( bcvB.getShortTOCName( 'GEN' ) ) )
print( "Gen book abbreviation:", repr( bcvB.getBooknameAbbreviation( 'GEN' ) ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( bcvB )
if BibleOrgSysGlobals.strictCheckingFlag:
bcvB.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bcbibleErrors = bcvB.getErrors()
# print( bcbibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##bcvB.toDrupalBible()
bcvB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj )
else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion)
#testFolder = "Tests/DataFilesForTests/BCVTest1/"
testFolder = "OutputFiles/BOS_BCV_Export/"
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA1")
result1 = BCVBibleFileCheck(testFolder)
if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA1", result1)
if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA2")
result2 = BCVBibleFileCheck(testFolder,
autoLoad=True) # But doesn't preload books
if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA2", result2)
#result2.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result2.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result2.getErrors()
# print( bibleErrors )
#if BibleOrgSysGlobals.commandLineArguments.export:
###result2.toDrupalBible()
#result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
if BibleOrgSysGlobals.verbosityLevel > 0: print("\nBCV TestA3")
result3 = BCVBibleFileCheck(testFolder,
autoLoad=True,
autoLoadBooks=True)
if BibleOrgSysGlobals.verbosityLevel > 1: print("BCV TestA3", result3)
#result3.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result3.getErrors()
# print( bibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##result3.toDrupalBible()
result3.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
if 0: # all discovered modules in the test folder
foundFolders, foundFiles = [], []
for something in os.listdir(testFolder):
somepath = os.path.join(testFolder, something)
if os.path.isdir(somepath): foundFolders.append(something)
elif os.path.isfile(somepath): foundFiles.append(something)
if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work
if BibleOrgSysGlobals.verbosityLevel > 1:
print("\nTrying all {} discovered modules…".format(
len(foundFolders)))
parameters = [folderName for folderName in sorted(foundFolders)]
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses
) as pool: # start worker processes
results = pool.map(testBCV,
parameters) # have the pool do our loads
assert len(results) == len(
parameters
) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate(sorted(foundFolders)):
if BibleOrgSysGlobals.verbosityLevel > 1:
print("\nBCV D{}/ Trying {}".format(j + 1, someFolder))
#myTestFolder = os.path.join( testFolder, someFolder+'/' )
testBCV(someFolder)
if 0: # Load and process some of our test versions
count = 0
for name, encoding, testFolder in (
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest1/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/BCVTest2/"),
("Exported", 'utf-8', "Tests/BOS_BCV_Export/"),
):
count += 1
if os.access(testFolder, os.R_OK):
if BibleOrgSysGlobals.verbosityLevel > 0:
print("\nBCV A{}/".format(count))
bcvB = BCVBible(testFolder, name, encoding=encoding)
bcvB.load()
if BibleOrgSysGlobals.verbosityLevel > 1:
print("Gen assumed book name:",
repr(bcvB.getAssumedBookName('GEN')))
print("Gen long TOC book name:",
repr(bcvB.getLongTOCName('GEN')))
print("Gen short TOC book name:",
repr(bcvB.getShortTOCName('GEN')))
print("Gen book abbreviation:",
repr(bcvB.getBooknameAbbreviation('GEN')))
if BibleOrgSysGlobals.verbosityLevel > 0: print(bcvB)
if BibleOrgSysGlobals.strictCheckingFlag:
bcvB.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
bcbibleErrors = bcvB.getErrors()
# print( bcbibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##bcvB.toDrupalBible()
bcvB.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
newObj = BibleOrgSysGlobals.unpickleObject(
BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle',
os.path.join("OutputFiles/",
"BOS_Bible_Object_Pickle/"))
if BibleOrgSysGlobals.verbosityLevel > 0:
print("newObj is", newObj)
else:
print(
"\nSorry, test folder {!r} is not readable on this computer."
.format(testFolder))
def createOpenSongXML(BibleObject,
outputFolder=None,
controlDict=None,
validationSchema=None):
"""
Using settings from the given control file,
converts the USFM information to a UTF-8 OpenSong XML file.
This format is roughly documented at http://de.wikipedia.org/wiki/OpenSong_XML
but more fields can be discovered by looking at downloaded files.
"""
if BibleOrgSysGlobals.verbosityLevel > 1:
print("Running createOpenSongXML…")
if BibleOrgSysGlobals.debugFlag: assert BibleObject.books
ignoredMarkers, unhandledMarkers, unhandledBooks = set(), set(), []
def writeOpenSongBook(writerObject, BBB, bkData):
"""Writes a book to the OpenSong XML writerObject."""
#print( 'BIBLEBOOK', [('bnumber',BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber(BBB)), ('bname',BibleOrgSysGlobals.BibleBooksCodes.getEnglishName_NR(BBB)), ('bsname',BibleOrgSysGlobals.BibleBooksCodes.getOSISAbbreviation(BBB))] )
OSISAbbrev = BibleOrgSysGlobals.BibleBooksCodes.getOSISAbbreviation(
BBB)
if not OSISAbbrev:
logging.warning(
"toOpenSong: Can't write {} OpenSong book because no OSIS code available"
.format(BBB))
unhandledBooks.append(BBB)
return
writerObject.writeLineOpen('b', ('n', bkData.getAssumedBookNames()[0]))
haveOpenChapter, startedFlag, gotVP, accumulator = False, False, None, ""
C, V = '-1', '-1' # So first/id line starts at -1:0
for processedBibleEntry in bkData._processedLines: # Process internal Bible data lines
marker, text, extras = processedBibleEntry.getMarker(
), processedBibleEntry.getCleanText(
), processedBibleEntry.getExtras()
#print( marker, repr(text) )
#if text: assert text[0] != ' '
if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS:
continue # Just ignore added markers -- not needed here
if marker in USFM_PRECHAPTER_MARKERS:
if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag:
assert C == '-1' or marker == 'rem' or marker.startswith(
'mte')
V = str(int(V) + 1)
if marker in OFTEN_IGNORED_USFM_HEADER_MARKERS or marker in (
'ie', ): # Just ignore these lines
ignoredMarkers.add(marker)
elif marker == 'c':
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
accumulator = ''
if haveOpenChapter:
writerObject.writeLineClose('c')
C, V = text, '0'
writerObject.writeLineOpen('c', ('n', text))
haveOpenChapter = True
elif marker in (
'c#',
): # These are the markers that we can safely ignore for this export
ignoredMarkers.add(marker)
elif marker == 'vp#': # This precedes a v field and has the verse number to be printed
gotVP = text # Just remember it for now
elif marker == 'v':
V = text
if gotVP: # this is the verse number to be published
text = gotVP
gotVP = None
startedFlag = True
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
accumulator = ''
#print( "Text {!r}".format( text ) )
if not text:
logging.warning("createOpenSongXML: Missing text for v")
continue
verseNumberString = text.replace('<', '').replace(
'>', ''
).replace(
'"', ''
) # Used below but remove anything that'll cause a big XML problem later
elif marker in ('mt1','mt2','mt3','mt4', 'mte1','mte2','mte3','mte4', 'ms1','ms2','ms3','ms4', ) \
or marker in USFM_ALL_INTRODUCTION_MARKERS \
or marker in ('s1','s2','s3','s4', 'r','sr','mr', 'd','sp','cd', 'cl','lit', ):
ignoredMarkers.add(marker)
elif marker in USFM_BIBLE_PARAGRAPH_MARKERS:
if BibleOrgSysGlobals.debugFlag: assert not text and not extras
ignoredMarkers.add(marker)
elif marker in (
'b',
'nb',
'ib',
):
if BibleOrgSysGlobals.debugFlag: assert not text and not extras
ignoredMarkers.add(marker)
elif marker in (
'v~',
'p~',
):
if BibleOrgSysGlobals.debugFlag: assert text or extras
if not text: # this is an empty (untranslated) verse
text = '- - -' # but we'll put in a filler
if startedFlag:
accumulator += (' ' if accumulator else
'') + BibleOrgSysGlobals.makeSafeXML(text)
else:
if text:
logging.warning(
"toOpenSong: lost text in {} field in {} {}:{} {!r}".
format(marker, BBB, C, V, text))
#if BibleOrgSysGlobals.debugFlag: halt
if extras:
logging.warning(
"toOpenSong: lost extras in {} field in {} {}:{}".
format(marker, BBB, C, V))
#if BibleOrgSysGlobals.debugFlag: halt
unhandledMarkers.add(marker)
if extras and marker not in (
'v~',
'p~',
) and marker not in ignoredMarkers:
logging.critical(
"toOpenSong: extras not handled for {} at {} {}:{}".format(
marker, BBB, C, V))
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
if haveOpenChapter:
writerObject.writeLineClose('c')
writerObject.writeLineClose('b')
# end of createOpenSongXML.writeOpenSongBook
# Set-up our Bible reference system
if 'PublicationCode' not in controlDict or controlDict[
'PublicationCode'] == 'GENERIC':
BOS = BibleObject.genericBOS
BRL = BibleObject.genericBRL
else:
BOS = BibleOrganisationalSystem(controlDict['PublicationCode'])
BRL = BibleReferenceList(BOS, BibleObject=None)
if BibleOrgSysGlobals.verbosityLevel > 2:
print(_(" Exporting to OpenSong format…"))
try:
osOFn = controlDict['OpenSongOutputFilename']
except KeyError:
osOFn = 'Bible.osong'
filename = BibleOrgSysGlobals.makeSafeFilename(osOFn)
xw = MLWriter(filename, outputFolder)
xw.setHumanReadable()
xw.start()
xw.writeLineOpen('Bible')
for BBB, bookData in BibleObject.books.items():
writeOpenSongBook(xw, BBB, bookData)
xw.writeLineClose('Bible')
xw.close()
if ignoredMarkers:
logging.info("createOpenSongXML: Ignored markers were {}".format(
ignoredMarkers))
if BibleOrgSysGlobals.verbosityLevel > 2:
print(" " + _("WARNING: Ignored createOpenSongXML markers were {}"
).format(ignoredMarkers))
if unhandledMarkers:
logging.warning("createOpenSongXML: Unhandled markers were {}".format(
unhandledMarkers))
if BibleOrgSysGlobals.verbosityLevel > 1:
print(" " +
_("WARNING: Unhandled toOpenSong markers were {}").format(
unhandledMarkers))
if unhandledBooks:
logging.warning("createOpenSongXML: Unhandled books were {}".format(
unhandledBooks))
if BibleOrgSysGlobals.verbosityLevel > 1:
print(" " + _("WARNING: Unhandled createOpenSongXML books were {}"
).format(unhandledBooks))
# Now create a zipped version
filepath = os.path.join(outputFolder, filename)
if BibleOrgSysGlobals.verbosityLevel > 2:
print(" Zipping {} OpenSong file…".format(filename))
zf = zipfile.ZipFile(filepath + '.zip',
'w',
compression=zipfile.ZIP_DEFLATED)
zf.write(filepath, filename)
zf.close()
if validationSchema: return xw.validate(validationSchema)
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
print(" createOpenSongXML finished successfully.")
return True
def createMySwordModule( self, outputFolder, controlDict ):
"""
Create a SQLite3 database module for the program MySword.
self here is a Bible object with _processedLines
"""
import tarfile
from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS
from theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine
def writeMSBook( sqlObject, BBB, ourGlobals ):
"""
Writes a book to the MySword sqlObject file.
"""
nonlocal lineCount
bkData = self.books[BBB] if BBB in self.books else None
#print( bkData._processedLines )
verseList = BOS.getNumVersesList( BBB )
nBBB = BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber( BBB )
numC, numV = len(verseList), verseList[0]
ourGlobals['line'], ourGlobals['lastLine'] = '', None
ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals['pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals['pi7'] = False
if bkData:
# Write book headings (stuff before chapter 1)
ourGlobals['line'] = theWordHandleIntroduction( BBB, bkData, ourGlobals )
# Write the verses
C = V = 1
ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None
while True:
verseData = None
if bkData:
try:
result = bkData.getContextVerseData( (BBB,str(C),str(V),) )
verseData, context = result
except KeyError: # Missing verses
logging.warning( "BibleWriter.toMySword: missing source verse at {} {}:{}".format( BBB, C, V ) )
# Handle some common versification anomalies
if (BBB,C,V) == ('JN3',1,14): # Add text for v15 if it exists
try:
result15 = bkData.getContextVerseData( ('JN3','1','15',) )
verseData15, context15 = result15
verseData.extend( verseData15 )
except KeyError: pass # just ignore it
elif (BBB,C,V) == ('REV',12,17): # Add text for v15 if it exists
try:
result18 = bkData.getContextVerseData( ('REV','12','18',) )
verseData18, context18 = result18
verseData.extend( verseData18 )
except KeyError: pass # just ignore it
composedLine = ''
if verseData: composedLine = theWordComposeVerseLine( BBB, C, V, verseData, ourGlobals )
# Stay one line behind (because paragraph indicators get appended to the previous line)
if ourGlobals['lastBCV'] is not None \
and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
ourGlobals['lastLine'] = composedLine
ourGlobals['lastBCV'] = (nBBB,C,V)
V += 1
if V > numV:
C += 1
if C > numC:
break
else: # next chapter only
numV = verseList[C-1]
V = 1
#assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything
# Write the last line of the file
if ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
# end of toMySword.writeMSBook
# Set-up their Bible reference system
BOS = BibleOrganizationalSystem( 'GENERIC-KJV-66-ENG' )
#BRL = BibleReferenceList( BOS, BibleObject=None )
# Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.)
if len(self) <= (39+4) and self.containsAnyOT39Books() and not self.containsAnyNT27Books():
testament, startBBB, endBBB = 'OT', 'GEN', 'MAL'
booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines
elif len(self) <= (27+4) and self.containsAnyNT27Books() and not self.containsAnyOT39Books():
testament, startBBB, endBBB = 'NT', 'MAT', 'REV'
booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines
else: # assume it's an entire Bible
testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV'
booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines
extension = '.bbl.mybible'
if BibleOrgSysGlobals.verbosityLevel > 2: print( _(" Exporting to MySword format…") )
mySettings = {}
mySettings['unhandledMarkers'] = set()
handledBooks = []
if 'MySwordOutputFilename' in controlDict: filename = controlDict['MySwordOutputFilename']
elif self.sourceFilename: filename = self.sourceFilename
elif self.shortName: filename = self.shortName
elif self.abbreviation: filename = self.abbreviation
elif self.name: filename = self.name
else: filename = 'export'
if not filename.endswith( extension ): filename += extension # Make sure that we have the right file extension
filepath = os.path.join( outputFolder, BibleOrgSysGlobals.makeSafeFilename( filename ) )
if os.path.exists( filepath ): os.remove( filepath )
if BibleOrgSysGlobals.verbosityLevel > 2: print( " " + _("Writing {!r}…").format( filepath ) )
conn = sqlite3.connect( filepath )
cursor = conn.cursor()
# First write the settings Details table
exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete
customCSS = self.getSetting( 'CustomCSS' )
if customCSS: exeStr += ', CustomCSS TEXT'
exeStr += ')'
cursor.execute( exeStr )
values = []
description = self.getSetting( 'Description' )
if not description: description = self.getSetting( 'description' )
if not description: description = self.name
values.append( description )
if self.abbreviation: abbreviation = self.abbreviation
else: abbreviation = self.getSetting( 'WorkAbbreviation' )
if not abbreviation: abbreviation = self.name[:3].upper()
values.append( abbreviation )
comments = self.getSetting( 'Comments' )
values.append( comments )
version = self.getSetting( 'Version' )
values.append( version )
versionDate = self.getSetting( 'VersionDate' )
values.append( versionDate )
publishDate = self.getSetting( 'PublishDate' )
values.append( publishDate )
rightToLeft = self.getSetting( 'RightToLeft' )
values.append( rightToLeft )
values.append( True if testament=='OT' or testament=='BOTH' else False )
values.append( True if testament=='NT' or testament=='BOTH' else False )
Strong = self.getSetting( 'Strong' )
values.append( Strong if Strong else False )
if customCSS: values.append( customCSS )
exeStr = 'INSERT INTO "Details" VALUES(' + '?,'*(len(values)-1) + '?)'
#print( exeStr, values )
cursor.execute( exeStr, values )
#if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values )
#else: # Not debugging
#try: cursor.execute( exeStr, values )
#except sqlite3.InterfaceError:
#logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) )
# Now create and fill the Bible table
cursor.execute( 'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))' )
conn.commit() # save (commit) the changes
BBB, lineCount = startBBB, 0
while True: # Write each Bible book in the KJV order
writeMSBook( cursor, BBB, mySettings )
conn.commit() # save (commit) the changes
handledBooks.append( BBB )
if BBB == endBBB: break
BBB = BOS.getNextBookCode( BBB )
conn.commit() # save (commit) the changes
cursor.close()
if mySettings['unhandledMarkers']:
logging.warning( "BibleWriter.toMySword: Unhandled markers were {}".format( mySettings['unhandledMarkers'] ) )
if BibleOrgSysGlobals.verbosityLevel > 1:
print( " " + _("WARNING: Unhandled toMySword markers were {}").format( mySettings['unhandledMarkers'] ) )
unhandledBooks = []
for BBB in self.getBookList():
if BBB not in handledBooks: unhandledBooks.append( BBB )
if unhandledBooks:
logging.warning( "toMySword: Unhandled books were {}".format( unhandledBooks ) )
if BibleOrgSysGlobals.verbosityLevel > 1:
print( " " + _("WARNING: Unhandled toMySword books were {}").format( unhandledBooks ) )
# Now create the gzipped file
if BibleOrgSysGlobals.verbosityLevel > 2: print( " Compressing {} MySword file…".format( filename ) )
tar = tarfile.open( filepath+'.gz', 'w:gz' )
tar.add( filepath )
tar.close()
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
print( " BibleWriter.toMySword finished successfully." )
return True
def createMySwordModule(self, outputFolder, controlDict):
"""
Create a SQLite3 database module for the program MySword.
self here is a Bible object with _processedLines
"""
import tarfile
from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS
from theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine
def writeMSBook(sqlObject, BBB, ourGlobals):
"""
Writes a book to the MySword sqlObject file.
"""
nonlocal lineCount
bkData = self.books[BBB] if BBB in self.books else None
#print( bkData._processedLines )
verseList = BOS.getNumVersesList(BBB)
nBBB = BibleOrgSysGlobals.BibleBooksCodes.getReferenceNumber(BBB)
numC, numV = len(verseList), verseList[0]
ourGlobals['line'], ourGlobals['lastLine'] = '', None
ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals[
'pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals[
'pi7'] = False
if bkData:
# Write book headings (stuff before chapter 1)
ourGlobals['line'] = theWordHandleIntroduction(
BBB, bkData, ourGlobals)
# Write the verses
C = V = 1
ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None
while True:
verseData = None
if bkData:
try:
result = bkData.getContextVerseData((
BBB,
str(C),
str(V),
))
verseData, context = result
except KeyError: # Missing verses
logging.warning(
"BibleWriter.createMySwordModule: missing source verse at {} {}:{}"
.format(BBB, C, V))
# Handle some common versification anomalies
if (BBB, C, V) == ('JN3', 1,
14): # Add text for v15 if it exists
try:
result15 = bkData.getContextVerseData((
'JN3',
'1',
'15',
))
verseData15, context15 = result15
verseData.extend(verseData15)
except KeyError:
pass # just ignore it
elif (BBB, C, V) == ('REV', 12,
17): # Add text for v15 if it exists
try:
result18 = bkData.getContextVerseData((
'REV',
'12',
'18',
))
verseData18, context18 = result18
verseData.extend(verseData18)
except KeyError:
pass # just ignore it
composedLine = ''
if verseData:
composedLine = theWordComposeVerseLine(
BBB, C, V, verseData, ourGlobals)
# Stay one line behind (because paragraph indicators get appended to the previous line)
if ourGlobals['lastBCV'] is not None \
and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
ourGlobals['lastLine'] = composedLine
ourGlobals['lastBCV'] = (nBBB, C, V)
V += 1
if V > numV:
C += 1
if C > numC:
break
else: # next chapter only
numV = verseList[C - 1]
V = 1
#assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything
# Write the last line of the file
if ourGlobals[
'lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
# end of createMySwordModule.writeMSBook
# Set-up their Bible reference system
BOS = BibleOrganisationalSystem('GENERIC-KJV-66-ENG')
#BRL = BibleReferenceList( BOS, BibleObject=None )
# Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.)
if len(self) <= (39 + 4) and self.containsAnyOT39Books(
) and not self.containsAnyNT27Books():
testament, startBBB, endBBB = 'OT', 'GEN', 'MAL'
booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines
elif len(self) <= (27 + 4) and self.containsAnyNT27Books(
) and not self.containsAnyOT39Books():
testament, startBBB, endBBB = 'NT', 'MAT', 'REV'
booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines
else: # assume it's an entire Bible
testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV'
booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines
extension = '.bbl.mybible'
if BibleOrgSysGlobals.verbosityLevel > 2:
print(_(" Exporting to MySword format…"))
mySettings = {}
mySettings['unhandledMarkers'] = set()
handledBooks = []
if 'MySwordOutputFilename' in controlDict:
filename = controlDict['MySwordOutputFilename']
elif self.sourceFilename:
filename = self.sourceFilename
elif self.shortName:
filename = self.shortName
elif self.abbreviation:
filename = self.abbreviation
elif self.name:
filename = self.name
else:
filename = 'export'
if not filename.endswith(extension):
filename += extension # Make sure that we have the right file extension
filepath = os.path.join(outputFolder,
BibleOrgSysGlobals.makeSafeFilename(filename))
if os.path.exists(filepath): os.remove(filepath)
if BibleOrgSysGlobals.verbosityLevel > 2:
print(' createMySwordModule: ' + _("Writing {!r}…").format(filepath))
conn = sqlite3.connect(filepath)
cursor = conn.cursor()
# First write the settings Details table
exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete
customCSS = self.getSetting('CustomCSS')
if customCSS: exeStr += ', CustomCSS TEXT'
exeStr += ')'
cursor.execute(exeStr)
values = []
description = self.getSetting('Description')
if not description: description = self.getSetting('description')
if not description: description = self.name
values.append(description)
if self.abbreviation: abbreviation = self.abbreviation
else: abbreviation = self.getSetting('WorkAbbreviation')
if not abbreviation: abbreviation = self.name[:3].upper()
values.append(abbreviation)
comments = self.getSetting('Comments')
values.append(comments)
version = self.getSetting('Version')
values.append(version)
versionDate = self.getSetting('VersionDate')
values.append(versionDate)
publishDate = self.getSetting('PublishDate')
values.append(publishDate)
rightToLeft = self.getSetting('RightToLeft')
values.append(rightToLeft)
values.append(True if testament == 'OT' or testament == 'BOTH' else False)
values.append(True if testament == 'NT' or testament == 'BOTH' else False)
Strong = self.getSetting('Strong')
values.append(Strong if Strong else False)
if customCSS: values.append(customCSS)
exeStr = 'INSERT INTO "Details" VALUES(' + '?,' * (len(values) - 1) + '?)'
#print( exeStr, values )
cursor.execute(exeStr, values)
#if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values )
#else: # Not debugging
#try: cursor.execute( exeStr, values )
#except sqlite3.InterfaceError:
#logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) )
# Now create and fill the Bible table
cursor.execute(
'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))'
)
conn.commit() # save (commit) the changes
BBB, lineCount = startBBB, 0
while True: # Write each Bible book in the KJV order
writeMSBook(cursor, BBB, mySettings)
conn.commit() # save (commit) the changes
handledBooks.append(BBB)
if BBB == endBBB: break
BBB = BOS.getNextBookCode(BBB)
conn.commit() # save (commit) the changes
cursor.close()
if mySettings['unhandledMarkers']:
logging.warning(
"BibleWriter.createMySwordModule: Unhandled markers were {}".
format(mySettings['unhandledMarkers']))
if BibleOrgSysGlobals.verbosityLevel > 1:
print(" " +
_("WARNING: Unhandled createMySwordModule markers were {}"
).format(mySettings['unhandledMarkers']))
unhandledBooks = []
for BBB in self.getBookList():
if BBB not in handledBooks: unhandledBooks.append(BBB)
if unhandledBooks:
logging.warning("createMySwordModule: Unhandled books were {}".format(
unhandledBooks))
if BibleOrgSysGlobals.verbosityLevel > 1:
print(" " +
_("WARNING: Unhandled createMySwordModule books were {}"
).format(unhandledBooks))
# Now create the gzipped file
if BibleOrgSysGlobals.verbosityLevel > 2:
print(" Compressing {} MySword file…".format(filename))
tar = tarfile.open(filepath + '.gz', 'w:gz')
tar.add(filepath)
tar.close()
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
print(" BibleWriter.createMySwordModule finished successfully.")
return True
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print( ProgNameVersion )
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
for j,testFolder in enumerate( ("Tests/DataFilesForTests/USFMTest1/",
"Tests/DataFilesForTests/USFMTest2/",
"Tests/DataFilesForTests/USFMTest3/",
"Tests/DataFilesForTests/USFMAllMarkersProject/",
"Tests/DataFilesForTests/USFMErrorProject/",
"Tests/DataFilesForTests/PTX7Test/",
"OutputFiles/BOS_USFM_Export/",
"OutputFiles/BOS_USFM_Reexport/",
"MadeUpFolder/",
) ):
print( "\nUSFM A{} testfolder is: {}".format( j+1, testFolder ) )
result1 = USFMBibleFileCheck( testFolder )
if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAa", result1 )
result2 = USFMBibleFileCheck( testFolder, autoLoad=True )
if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAb", result2 )
result3 = USFMBibleFileCheck( testFolder, autoLoadBooks=True )
if BibleOrgSysGlobals.verbosityLevel > 1: print( "USFM TestAc", result3 )
if isinstance( result3, Bible ):
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
UsfmBErrors = UsfmB.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
result3.pickle()
##UsfmB.toDrupalBible()
result3.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
if 1: # Load and process some of our test versions
for j,(name, encoding, testFolder) in enumerate( (
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest1/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest2/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest3/"),
("WEB+", 'utf-8', "Tests/DataFilesForTests/USFMAllMarkersProject/"),
("UEP", 'utf-8', "Tests/DataFilesForTests/USFMErrorProject/"),
("Exported", 'utf-8', "Tests/BOS_USFM_Export/"),
) ):
if os.access( testFolder, os.R_OK ):
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nUSFM B{}/".format( j+1 ) )
UsfmB = USFMBible( testFolder, name, encoding=encoding )
UsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 1:
print( "Gen assumed book name:", repr( UsfmB.getAssumedBookName( 'GEN' ) ) )
print( "Gen long TOC book name:", repr( UsfmB.getLongTOCName( 'GEN' ) ) )
print( "Gen short TOC book name:", repr( UsfmB.getShortTOCName( 'GEN' ) ) )
print( "Gen book abbreviation:", repr( UsfmB.getBooknameAbbreviation( 'GEN' ) ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( UsfmB )
if BibleOrgSysGlobals.strictCheckingFlag:
UsfmB.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
UsfmBErrors = UsfmB.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
UsfmB.pickle()
##UsfmB.toDrupalBible()
UsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join( "OutputFiles/", "BOS_Bible_Object_Pickle/" ) )
if BibleOrgSysGlobals.verbosityLevel > 0: print( "newObj is", newObj )
elif BibleOrgSysGlobals.verbosityLevel > 0:
print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
if 0: # Test a whole folder full of folders of USFM Bibles
testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/"
def findInfo( somepath ):
""" Find out info about the project from the included copyright.htm file """
cFilepath = os.path.join( somepath, "copyright.htm" )
if not os.path.exists( cFilepath ): return
with open( cFilepath ) as myFile: # Automatically closes the file when done
lastLine, lineCount = None, 0
title, nameDict = None, {}
for line in myFile:
lineCount += 1
if lineCount==1:
if line[0]==chr(65279): #U+FEFF
logging.info( "USFMBible.findInfo1: Detected Unicode Byte Order Marker (BOM) in {}".format( "copyright.htm" ) )
line = line[1:] # Remove the UTF-16 Unicode Byte Order Marker (BOM)
elif line[:3] == '': # 0xEF,0xBB,0xBF
logging.info( "USFMBible.findInfo2: Detected Unicode Byte Order Marker (BOM) in {}".format( "copyright.htm" ) )
line = line[3:] # Remove the UTF-8 Unicode Byte Order Marker (BOM)
if line[-1]=='\n': line = line[:-1] # Removing trailing newline character
if not line: continue # Just discard blank lines
lastLine = line
if line.startswith("<title>"): title = line.replace("<title>","").replace("</title>","").strip()
if line.startswith('<option value="'):
adjLine = line.replace('<option value="','').replace('</option>','')
USFM_BBB, name = adjLine[:3], adjLine[11:]
BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromUSFM( USFM_BBB )
#print( USFM_BBB, BBB, name )
nameDict[BBB] = name
return title, nameDict
# end of findInfo
count = totalBooks = 0
if os.access( testBaseFolder, os.R_OK ): # check that we can read the test data
for something in sorted( os.listdir( testBaseFolder ) ):
somepath = os.path.join( testBaseFolder, something )
if os.path.isfile( somepath ): print( "Ignoring file {!r} in {!r}".format( something, testBaseFolder ) )
elif os.path.isdir( somepath ): # Let's assume that it's a folder containing a USFM (partial) Bible
#if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules
count += 1
title = None
findInfoResult = findInfo( somepath )
if findInfoResult: title, bookNameDict = findInfoResult
if title is None: title = something[:-5] if something.endswith("_usfm") else something
name, encoding, testFolder = title, 'utf-8', somepath
if os.access( testFolder, os.R_OK ):
if BibleOrgSysGlobals.verbosityLevel > 0: print( "\nUSFM C{}/".format( count ) )
UsfmB = USFMBible( testFolder, name, encoding=encoding )
UsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 0: print( UsfmB )
if BibleOrgSysGlobals.strictCheckingFlag:
UsfmB.check()
UsfmBErrors = UsfmB.getErrors()
#print( UsfmBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
UsfmB.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
else: print( "\nSorry, test folder {!r} is not readable on this computer.".format( testFolder ) )
if count: print( "\n{} total USFM (partial) Bibles processed.".format( count ) )
if totalBooks: print( "{} total books ({} average per folder)".format( totalBooks, round(totalBooks/count) ) )
elif BibleOrgSysGlobals.verbosityLevel > 0:
print( "\nSorry, test folder {!r} is not readable on this computer.".format( testBaseFolder ) )
def demo():
"""
Demonstrate reading and checking some Bible databases.
"""
if BibleOrgSysGlobals.verbosityLevel > 0: print(ProgNameVersion)
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
for j, testFolder in enumerate((
"Tests/DataFilesForTests/USFMTest1/",
"Tests/DataFilesForTests/USFMTest2/",
"Tests/DataFilesForTests/USFMTest3/",
"Tests/DataFilesForTests/USFMAllMarkersProject/",
"Tests/DataFilesForTests/USFMErrorProject/",
"Tests/DataFilesForTests/PTX7Test/",
"OutputFiles/BOS_USFM_Export/",
"OutputFiles/BOS_USFM_Reexport/",
"MadeUpFolder/",
)):
print("\nUSFM A{} testfolder is: {}".format(j + 1, testFolder))
result1 = USFMBibleFileCheck(testFolder)
if BibleOrgSysGlobals.verbosityLevel > 1:
print("USFM TestAa", result1)
result2 = USFMBibleFileCheck(testFolder, autoLoad=True)
if BibleOrgSysGlobals.verbosityLevel > 1:
print("USFM TestAb", result2)
result3 = USFMBibleFileCheck(testFolder, autoLoadBooks=True)
if BibleOrgSysGlobals.verbosityLevel > 1:
print("USFM TestAc", result3)
if isinstance(result3, Bible):
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#print( result3.books['GEN']._processedLines[0:40] )
UsfmBErrors = result3.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
result3.pickle()
##result3.toDrupalBible()
result3.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
if 1: # Load and process some of our test versions
for j, (name, encoding, testFolder) in enumerate((
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest1/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest2/"),
("Matigsalug", 'utf-8', "Tests/DataFilesForTests/USFMTest3/"),
("WEB+", 'utf-8',
"Tests/DataFilesForTests/USFMAllMarkersProject/"),
("UEP", 'utf-8', "Tests/DataFilesForTests/USFMErrorProject/"),
("Exported", 'utf-8', "Tests/BOS_USFM_Export/"),
)):
if os.access(testFolder, os.R_OK):
if BibleOrgSysGlobals.verbosityLevel > 0:
print("\nUSFM B{}/".format(j + 1))
UsfmB = USFMBible(testFolder, name, encoding=encoding)
UsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 1:
print("Gen assumed book name:",
repr(UsfmB.getAssumedBookName('GEN')))
print("Gen long TOC book name:",
repr(UsfmB.getLongTOCName('GEN')))
print("Gen short TOC book name:",
repr(UsfmB.getShortTOCName('GEN')))
print("Gen book abbreviation:",
repr(UsfmB.getBooknameAbbreviation('GEN')))
if BibleOrgSysGlobals.verbosityLevel > 0: print(UsfmB)
if BibleOrgSysGlobals.strictCheckingFlag:
UsfmB.check()
#print( UsfmB.books['GEN']._processedLines[0:40] )
UsfmBErrors = UsfmB.getErrors()
# print( UBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
UsfmB.pickle()
##UsfmB.toDrupalBible()
UsfmB.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
newObj = BibleOrgSysGlobals.unpickleObject(
BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle',
os.path.join("OutputFiles/",
"BOS_Bible_Object_Pickle/"))
if BibleOrgSysGlobals.verbosityLevel > 0:
print("newObj is", newObj)
elif BibleOrgSysGlobals.verbosityLevel > 0:
print(
"\nSorry, test folder {!r} is not readable on this computer."
.format(testFolder))
if 0: # Test a whole folder full of folders of USFM Bibles
testBaseFolder = "Tests/DataFilesForTests/theWordRoundtripTestFiles/"
def findInfo(somepath):
""" Find out info about the project from the included copyright.htm file """
cFilepath = os.path.join(somepath, "copyright.htm")
if not os.path.exists(cFilepath): return
with open(cFilepath
) as myFile: # Automatically closes the file when done
lastLine, lineCount = None, 0
title, nameDict = None, {}
for line in myFile:
lineCount += 1
if lineCount == 1:
if line[0] == chr(65279): #U+FEFF
logging.info(
"USFMBible.findInfo1: Detected Unicode Byte Order Marker (BOM) in {}"
.format("copyright.htm"))
line = line[
1:] # Remove the UTF-16 Unicode Byte Order Marker (BOM)
elif line[:3] == '': # 0xEF,0xBB,0xBF
logging.info(
"USFMBible.findInfo2: Detected Unicode Byte Order Marker (BOM) in {}"
.format("copyright.htm"))
line = line[
3:] # Remove the UTF-8 Unicode Byte Order Marker (BOM)
if line[-1] == '\n':
line = line[:-1] # Removing trailing newline character
if not line: continue # Just discard blank lines
lastLine = line
if line.startswith("<title>"):
title = line.replace("<title>",
"").replace("</title>",
"").strip()
if line.startswith('<option value="'):
adjLine = line.replace('<option value="',
'').replace('</option>', '')
USFM_BBB, name = adjLine[:3], adjLine[11:]
BBB = BibleOrgSysGlobals.BibleBooksCodes.getBBBFromUSFM(
USFM_BBB)
#print( USFM_BBB, BBB, name )
nameDict[BBB] = name
return title, nameDict
# end of findInfo
count = totalBooks = 0
if os.access(testBaseFolder,
os.R_OK): # check that we can read the test data
for something in sorted(os.listdir(testBaseFolder)):
somepath = os.path.join(testBaseFolder, something)
if os.path.isfile(somepath):
print("Ignoring file {!r} in {!r}".format(
something, testBaseFolder))
elif os.path.isdir(
somepath
): # Let's assume that it's a folder containing a USFM (partial) Bible
#if not something.startswith( 'ssx' ): continue # This line is used for debugging only specific modules
count += 1
title = None
findInfoResult = findInfo(somepath)
if findInfoResult: title, bookNameDict = findInfoResult
if title is None:
title = something[:-5] if something.endswith(
"_usfm") else something
name, encoding, testFolder = title, 'utf-8', somepath
if os.access(testFolder, os.R_OK):
if BibleOrgSysGlobals.verbosityLevel > 0:
print("\nUSFM C{}/".format(count))
UsfmB = USFMBible(testFolder, name, encoding=encoding)
UsfmB.load()
if BibleOrgSysGlobals.verbosityLevel > 0: print(UsfmB)
if BibleOrgSysGlobals.strictCheckingFlag:
UsfmB.check()
UsfmBErrors = UsfmB.getErrors()
#print( UsfmBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
UsfmB.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
else:
print(
"\nSorry, test folder {!r} is not readable on this computer."
.format(testFolder))
if count:
print("\n{} total USFM (partial) Bibles processed.".format(
count))
if totalBooks:
print("{} total books ({} average per folder)".format(
totalBooks, round(totalBooks / count)))
elif BibleOrgSysGlobals.verbosityLevel > 0:
print(
"\nSorry, test folder {!r} is not readable on this computer.".
format(testBaseFolder))
def __init__(self,
parameterOne,
resourcesObject=None,
downloadAllBooks=False):
"""
Create the Door43 cataloged Bible object.
parameterOne can be:
a catalog dictionary entry (and second parameter must be None)
or
an index into the BibleList in the resourcesObject passed as the second parameter
"""
if BibleOrgSysGlobals.debugFlag and debuggingThisModule:
print(
f"DCSBible.__init__( {parameterOne}, {resourcesObject}, {downloadAllBooks} )…"
)
if isinstance(parameterOne, dict):
assert resourcesObject is None
resourceDict = parameterOne
else:
assert isinstance(parameterOne, int)
assert resourcesObject # why ??? and isinstance( resourcesObject, Door43CatalogResources )
resourceDict = resourcesObject.getBibleResourceDict(parameterOne)
assert resourceDict and isinstance(resourceDict, dict)
#print( 'resourceDict', resourceDict )
#print( 'resourceDict', resourceDict.keys() )
self.baseURL = resourceDict['html_url']
#print( 'self.baseURL', self.baseURL )
adjustedRepoName = resourceDict['full_name'].replace('/', '--')
#print( 'adjustedRepoName', adjustedRepoName )
desiredFolderName = BibleOrgSysGlobals.makeSafeFilename(
adjustedRepoName)
unzippedFolderPath = os.path.join(
BibleOrgSysGlobals.DOWNLOADED_RESOURCES_FOLDER,
'Door43ContentServiceOnline/', f"{adjustedRepoName}/")
if downloadAllBooks:
# See if files already exist and are current (so don't download again)
alreadyDownloadedFlag = False
if os.path.isdir(unzippedFolderPath):
#print( f"Issued: {resourceDict['issued']}" )
updatedDatetime = datetime.strptime(resourceDict['updated_at'],
'%Y-%m-%dT%H:%M:%SZ')
#print( f"updatedDatetime: {updatedDatetime}" )
#print( f"folder: {os.stat(unzippedFolderPath).st_mtime}" )
folderModifiedDatetime = datetime.fromtimestamp(
os.stat(unzippedFolderPath).st_mtime)
#print( f"folderModifiedDatetime: {folderModifiedDatetime}" )
alreadyDownloadedFlag = folderModifiedDatetime > updatedDatetime
#print( f"alreadyDownloadedFlag: {alreadyDownloadedFlag}" )
if alreadyDownloadedFlag:
if BibleOrgSysGlobals.verbosityLevel > 1:
print(
"Skipping download because folder '{}' already exists."
.format(unzippedFolderPath))
else: # Download the zip file (containing all the USFM files, README.md, LICENSE.md, manifest.yaml, etc.)
# TODO: Change to .tar.gz instead of zip
zipURL = self.baseURL + '/archive/master.zip' # '/archive/master.tar.gz'
if BibleOrgSysGlobals.verbosityLevel > 1:
print("Downloading entire repo from '{}'…".format(zipURL))
try:
HTTPResponseObject = urllib.request.urlopen(zipURL)
except urllib.error.URLError as err:
#errorClass, exceptionInstance, traceback = sys.exc_info()
#print( '{!r} {!r} {!r}'.format( errorClass, exceptionInstance, traceback ) )
logging.critical("DCS URLError '{}' from {}".format(
err, zipURL))
return
# print( " HTTPResponseObject", HTTPResponseObject )
contentType = HTTPResponseObject.info().get('content-type')
if BibleOrgSysGlobals.debugFlag and debuggingThisModule:
print(" contentType", repr(contentType))
if contentType == 'application/octet-stream':
try:
os.makedirs(unzippedFolderPath)
except FileExistsError:
pass
downloadedData = HTTPResponseObject.read()
if BibleOrgSysGlobals.verbosityLevel > 0:
print(
f" Downloaded {len(downloadedData):,} bytes from '{zipURL}'"
)
# Bug in Python up to 3.7 makes this not work for large aligned Bibles (3+ MB)
# myTempFile = tempfile.SpooledTemporaryFile()
myTempFile = tempfile.TemporaryFile()
myTempFile.write(downloadedData)
with zipfile.ZipFile(myTempFile) as myzip:
# NOTE: Could be a security risk here
myzip.extractall(unzippedFolderPath)
myTempFile.close() # Automatically deletes the file
else:
print(" contentType", repr(contentType))
halt # unknown content type
self.downloadedAllBooks = True
# There's probably a folder inside this folder
folders = os.listdir(unzippedFolderPath)
#print( 'folders', folders )
assert len(
folders
) == 1 # else maybe a previous download failed -- just manually delete the folder
desiredFolderName = folders[0] + '/'
#print( 'desiredFolderName', desiredFolderName )
USFMBible.__init__(self,
os.path.join(unzippedFolderPath,
desiredFolderName),
givenName=resourceDict['name'])
else:
self.downloadedAllBooks = False
self.attemptedDownload = {}
try:
os.makedirs(unzippedFolderPath)
except FileExistsError:
pass
USFMBible.__init__(self,
unzippedFolderPath,
givenName=resourceDict['name'])
self.objectNameString = 'DCS USFM Bible object'
def createEasyWorshipBible( BibleObject, outputFolder=None ):
"""
Write the pseudo USFM out into the compressed EasyWorship format.
Since we don't have a specification for the format,
and since we don't know the meaning of all the binary pieces of the file,
we can't be certain yet that this output will actually work. :-(
"""
from InternalBibleInternals import BOS_ADDED_NESTING_MARKERS
import zipfile
# It seems 7-9 give the correct two header bytes
ZLIB_COMPRESSION_LEVEL = 9 # -1=default(=6), 0=none, 1=fastest...9=highest compression level
if BibleOrgSysGlobals.verbosityLevel > 1: print( "Running createEasyWorshipBible…" )
if BibleOrgSysGlobals.debugFlag: assert BibleObject.books
if not BibleObject.doneSetupGeneric: BibleObject.__setupWriter()
if not outputFolder: outputFolder = 'OutputFiles/BOS_EasyWorshipBible_Export/'
if not os.access( outputFolder, os.F_OK ): os.makedirs( outputFolder ) # Make the empty folder if there wasn't already one there
# Set-up their Bible reference system
BOS = BibleOrganizationalSystem( 'GENERIC-KJV-66-ENG' )
ignoredMarkers = set()
# Before we write the file, let's compress all our books
# Books are written as C:V verseText with double-spaced lines
compressedDictionary = {}
for BBB,bookObject in BibleObject.books.items():
if BBB in ('FRT','INT','BAK','OTH','GLS','XXA','XXB','XXC','XXD','XXE','XXF','XXG',): continue # Ignore these books
pseudoESFMData = bookObject._processedLines
textBuffer = ''
vBridgeStartInt = vBridgeEndInt = None # For printing missing (bridged) verse numbers
for entry in pseudoESFMData:
marker, text = entry.getMarker(), entry.getCleanText()
#print( BBB, marker, text )
if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS: continue # Just ignore added markers -- not needed here
elif marker == 'c':
C = int( text ) # Just so we get an error if we have something different
V = lastVWritten = '0'
elif marker == 'v':
#V = text.replace( '–', '-' ).replace( '—', '-' ) # Replace endash, emdash with hyphen
V = text
for bridgeChar in ('-', '–', '—'): # hyphen, endash, emdash
ix = V.find( bridgeChar )
if ix != -1:
if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.verbosityLevel>2:
print( "createEasyWorshipBible: preparing for verse bridge in {} at {} {}:{}" \
.format( BibleObject.abbreviation, BBB, C, V ) )
# Remove verse bridges
vStart = V[:ix].replace( 'a', '' ).replace( 'b', '' ).replace( 'c', '' )
vEnd = V[ix+1:].replace( 'a', '' ).replace( 'b', '' ).replace( 'c', '' )
#print( BBB, repr(vStart), repr(vEnd) )
try: vBridgeStartInt, vBridgeEndInt = int( vStart ), int( vEnd )
except ValueError:
print( "createEasyWorshipBible: bridge doesn't seem to be integers in {} {}:{!r}".format( BBB, C, V ) )
vBridgeStartInt = vBridgeEndInt = None # One of them isn't an integer
#print( ' ', BBB, repr(vBridgeStartInt), repr(vBridgeEndInt) )
VBridgedText = V
V = vStart
break
elif marker == 'v~':
try:
if int(V) <= int(lastVWritten):
# TODO: Not sure what level the following should be? info/warning/error/critical ????
logging.warning( 'createEasyWorshipBible: Maybe duplicating {} {}:{} after {} with {}'.format( BBB, C, V, lastVWritten, text ) )
#continue
except ValueError: pass # had a verse bridge
if vBridgeStartInt and vBridgeEndInt: # We had a verse bridge
if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.verbosityLevel>2:
print( "createEasyWorshipBible: handling verse bridge in {} at {} {}:{}-{}" \
.format( BibleObject.abbreviation, BBB, C, vBridgeStartInt, vBridgeEndInt ) )
if 1: # new code -- copies the bridged text to all verses
for vNum in range( vBridgeStartInt, vBridgeEndInt+1 ): # Fill in missing verse numbers
textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} ({}) {}'.format( C, vNum, VBridgedText, text )
else: # old code
textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} ({}) {}'.format( C, vBridgeStartInt, vBridgeEndInt, text )
for vNum in range( vBridgeStartInt+1, vBridgeEndInt+1 ): # Fill in missing verse numbers
textBuffer += '\r\n\r\n{}:{} (-)'.format( C, vNum )
lastVWritten = str( vBridgeEndInt )
vBridgeStartInt = vBridgeEndInt = None
else:
textBuffer += ('\r\n\r\n' if textBuffer else '') + '{}:{} {}'.format( C, V, text )
lastVWritten = V
elif marker == 'p~':
if BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag:
assert textBuffer # This is a continued part of the verse -- failed with this bad source USFM:
# \c 1 \v 1 \p These events happened...
textBuffer += ' {}'.format( text ) # continuation of the same verse
else:
ignoredMarkers.add( marker )
#print( BBB, textBuffer )
textBuffer = textBuffer \
.replace( '“', '"' ).replace( '”', '"' ) \
.replace( "‘", "'" ).replace( "’", "'" ) \
.replace( '–', '--' ).replace( '—', '--' )
bookBytes = zlib.compress( textBuffer.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL )
#print( BBB, hexlify(bookBytes[:20]), bookBytes )
assert bookBytes[0]==0x78 and bookBytes[1]==0xda # Zlib compression header
appendage = b'QK\x03\x04' + struct.pack( '<I', len(textBuffer) ) + b'\x08\x00'
#print( "appendage", len(appendage), hexlify(appendage), appendage )
assert len(appendage) == 10
compressedDictionary[BBB] = bookBytes + appendage
# Work out the "compressed" (osfuscated) module name
#name = BibleObject.getAName()
##print( 'sn', repr(BibleObject.shortName) )
#if len(name)>18:
#if BibleObject.shortName: name = shortName
#elif name.endswith( ' Version' ): name = name[:-8]
#name = name.replace( ' ', '' )
#if not name.startswith( 'ezFree' ): name = 'ezFree' + name
name = 'ezFree' + ( BibleObject.abbreviation if BibleObject.abbreviation else 'UNK' )
if len(name)>16: name = name[:16] # Shorten
encodedNameBytes = zlib.compress( name.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL )
if BibleOrgSysGlobals.debugFlag:
print( 'Name {!r} went from {} to {} bytes'.format( name, len(name), len(encodedNameBytes) ) )
assert encodedNameBytes[0]==0x78 and encodedNameBytes[1]==0xda # Zlib compression header
assert len(encodedNameBytes) <= 26
filename = '{}{}'.format( BibleObject.abbreviation, FILENAME_ENDING ).lower()
filepath = os.path.join( outputFolder, BibleOrgSysGlobals.makeSafeFilename( filename ) )
if BibleOrgSysGlobals.verbosityLevel > 2: print( ' createEasyWorshipBible: ' + _("Writing {!r}…").format( filepath ) )
bookAddress = startingBookAddress = 14872 + len(name) + 18 + 4 # Name is something like ezFreeXXX
vBridgeStartInt = vBridgeEndInt = None # For printing missing (bridged) verse numbers
with open( filepath, 'wb' ) as myFile:
assert myFile.tell() == 0
# Write the header info to binary file
myFile.write( b'EasyWorship Bible Text\x1a\x02<\x00\x00\x00\xe0\x00\x00\x00' )
assert myFile.tell() == 32
nameBytes = ( BibleObject.getAName() ).encode( 'utf8' )
myFile.write( nameBytes + b'\x00' * (56 - len(nameBytes)) )
assert myFile.tell() == 88 # 32 + 56
# Write the numChapters,numVerses info along with the file position and length
for BBB in BOS.getBookList():
#bookName = BibleObject.getAssumedBookName( BBB )
try: bookName = BibleObject.books[BBB].shortTOCName
except (KeyError,AttributeError): bookName = None # KeyError if no BBB, AttributeError if no shortTOCName
#print( len(bookName) if bookName else '', bookName )
assert bookName is None or len(bookName) <= 51
if bookName: bookNameBytes = bookName.encode( 'utf8' )
else: bookNameBytes = b'' # Not compulsory -- will default to English
myFile.write( bookNameBytes + b'\x00' * (51 - len(bookNameBytes)) )
numVersesList = BOS.getNumVersesList( BBB )
numChapters = len( numVersesList )
myFile.write( struct.pack( 'B', numChapters ) )
for verseCount in numVersesList: myFile.write( struct.pack( 'B', verseCount ) )
myFile.write( b'\x00' * (157 - numChapters - 1) )
try: bookBytes = compressedDictionary[BBB] # if it exists
except KeyError: # Fill in missing books
missingString = "1:1 Book not available\r\n\r\n"
bookBytes = zlib.compress( missingString.encode( 'utf8' ), ZLIB_COMPRESSION_LEVEL )
assert bookBytes[0]==0x78 and bookBytes[1]==0xda # Zlib compression header
appendage = b'QK\x03\x04' + struct.pack( '<I', len(missingString) ) + b'\x08\x00'
assert len(appendage) == 10
bookBytes += appendage
compressedDictionary[BBB] = bookBytes
myFile.write( struct.pack( '<Q', bookAddress ) )
myFile.write( struct.pack( '<Q', len(bookBytes) ) )
bookAddress += len(bookBytes)
assert myFile.tell() == 14872 # 32 + 56 + 224*66
# Write the "compressed" (osfuscated) module name
myFile.write( struct.pack( '<I', len(name) + 18 ) )
assert myFile.tell() == 14876 # 32 + 56 + 224*66 + 4
myFile.write( encodedNameBytes )
appendage = b'QK\x03\x04' + struct.pack( 'B', len(name) ) + b'\x00'
#print( "appendage", len(appendage), hexlify(appendage), appendage )
assert len(appendage) == 6
myFile.write( appendage )
remainderCount = 18 + len(name) - len(encodedNameBytes) - 4 - len(appendage)
#print( "remainderCount", remainderCount )
assert remainderCount == 0
#myFile.write( b'\x00' * remainderCount )
myFile.write( b'\x00\x00\x08\x00' ) # Not sure what this means
#if debuggingThisModule or BibleOrgSysGlobals.debugFlag:
#print( "At", myFile.tell(), 'want', startingBookAddress )
assert myFile.tell() == startingBookAddress
# Write the book info to the binary files
for BBB in BOS.getBookList():
if BBB in compressedDictionary:
myFile.write( compressedDictionary[BBB] ) # Write zlib output
elif BibleOrgSysGlobals.verbosityLevel > 2:
print( ' Book {} is not available for EasyWorship export'.format( BBB ) )
# Write the end of file stuff
myFile.write( b'\x18:\x00\x00\x00\x00\x00\x00ezwBible' )
if ignoredMarkers:
logging.info( "createEasyWorshipBible: Ignored markers were {}".format( ignoredMarkers ) )
if BibleOrgSysGlobals.verbosityLevel > 2:
print( " " + _("WARNING: Ignored createEasyWorshipBible markers were {}").format( ignoredMarkers ) )
# Now create a zipped version
filepath = os.path.join( outputFolder, filename )
if BibleOrgSysGlobals.verbosityLevel > 2: print( " Zipping {} EWB file…".format( filename ) )
zf = zipfile.ZipFile( filepath+'.zip', 'w', compression=zipfile.ZIP_DEFLATED )
zf.write( filepath, filename )
zf.close()
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
print( " BibleWriter.createEasyWorshipBible finished successfully." )
return True
