def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
testFolderpath = Path('/mnt/SSDs/Bibles/unfoldingWordHelps/en_obs/')
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA1")
result1 = uWOBSBibleFileCheck(testFolderpath)
vPrint('Normal', debuggingThisModule, "uW OBS TestA1", result1)
vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA2")
result2 = uWOBSBibleFileCheck(
testFolderpath, autoLoad=True) # But doesn't preload books
vPrint('Normal', debuggingThisModule, "uW OBS TestA2", result2)
#result2.loadMetadataFile( os.path.join( testFolderpath, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result2.check()
#dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result2.getCheckResults()
#dPrint( 'Quiet', debuggingThisModule, bibleErrors )
#if BibleOrgSysGlobals.commandLineArguments.export:
###result2.toDrupalBible()
#result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False )
vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA3")
result3 = uWOBSBibleFileCheck(testFolderpath,
autoLoad=True,
autoLoadBooks=True)
vPrint('Normal', debuggingThisModule, "uW OBS TestA3", result3)
#result3.loadMetadataFile( os.path.join( testFolderpath, "BooknamesMetadata.txt" ) )
for BBB in ('OBS', 'RUT', 'JN3'):
vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gCVD",
result3.getContextVerseData(('OBS', '1', '1', '')))
vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gVDL",
result3.getVerseDataList(('OBS', '1', '1', '')))
vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gVT",
result3.getVerseText(('OBS', '1', '1', '')))
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] )
bibleErrors = result3.getCheckResults()
#dPrint( 'Quiet', debuggingThisModule, bibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##result3.toDrupalBible()
result3.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
if 0: # all discovered modules in the test folder
foundFolders, foundFiles = [], []
for something in os.listdir(testFolderpath):
somepath = os.path.join(testFolderpath, something)
if os.path.isdir(somepath): foundFolders.append(something)
elif os.path.isfile(somepath): foundFiles.append(something)
if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work
vPrint(
'Normal', debuggingThisModule,
"\nTrying all {} discovered modules…".format(
len(foundFolders)))
parameters = [folderName for folderName in sorted(foundFolders)]
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses
) as pool: # start worker processes
results = pool.map(testBCV,
parameters) # have the pool do our loads
assert len(results) == len(
parameters
) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate(sorted(foundFolders)):
vPrint('Normal', debuggingThisModule,
"\nuW OBS D{}/ Trying {}".format(j + 1, someFolder))
#myTestFolder = os.path.join( testFolderpath, someFolder+'/' )
testBCV(someFolder)
if 0: # Load and process some of our test versions
count = 0
for name, encoding, testFolder in (
("Matigsalug", 'utf-8',
BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('BCVTest1/')
),
("Matigsalug", 'utf-8',
BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('BCVTest2/')
),
("Exported", 'utf-8', "Tests/BOS_BCV_Export/"),
):
count += 1
if os.access(testFolder, os.R_OK):
vPrint('Quiet', debuggingThisModule,
"\nuW OBS A{}/".format(count))
uWnB = uWOBSBible(testFolder, name, encoding=encoding)
uWnB.load()
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint('Quiet', debuggingThisModule,
"Gen assumed book name:",
repr(uWnB.getAssumedBookName('GEN')))
vPrint('Quiet', debuggingThisModule,
"Gen long TOC book name:",
repr(uWnB.getLongTOCName('GEN')))
vPrint('Quiet', debuggingThisModule,
"Gen short TOC book name:",
repr(uWnB.getShortTOCName('GEN')))
vPrint('Quiet', debuggingThisModule,
"Gen book abbreviation:",
repr(uWnB.getBooknameAbbreviation('GEN')))
vPrint('Quiet', debuggingThisModule, uWnB)
if BibleOrgSysGlobals.strictCheckingFlag:
uWnB.check()
#dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] )
bcbibleErrors = uWnB.getCheckResults()
#dPrint( 'Quiet', debuggingThisModule, bcbibleErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##uWnB.toDrupalBible()
uWnB.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
newObj = BibleOrgSysGlobals.unpickleObject(
BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle',
os.path.join("BOSOutputFiles/",
"BOS_Bible_Object_Pickle/"))
vPrint('Quiet', debuggingThisModule, "newObj is", newObj)
else:
vPrint(
'Quiet', debuggingThisModule,
f"\nSorry, test folder '{testFolder}' is not readable on this computer."
)
def createOpenSongXML(BibleObject,
outputFolder=None,
controlDict=None,
validationSchema=None):
"""
Using settings from the given control file,
converts the USFM information to a UTF-8 OpenSong XML file.
This format is roughly documented at http://de.wikipedia.org/wiki/OpenSong_XML
but more fields can be discovered by looking at downloaded files.
"""
vPrint('Normal', debuggingThisModule, "Running createOpenSongXML…")
if BibleOrgSysGlobals.debugFlag: assert BibleObject.books
ignoredMarkers, unhandledMarkers, unhandledBooks = set(), set(), []
def writeOpenSongBook(writerObject, BBB: str, bkData):
"""Writes a book to the OpenSong XML writerObject."""
#dPrint( 'Quiet', debuggingThisModule, 'BIBLEBOOK', [('bnumber',BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB)), ('bname',BibleOrgSysGlobals.loadedBibleBooksCodes.getEnglishName_NR(BBB)), ('bsname',BibleOrgSysGlobals.loadedBibleBooksCodes.getOSISAbbreviation(BBB))] )
OSISAbbrev = BibleOrgSysGlobals.loadedBibleBooksCodes.getOSISAbbreviation(
BBB)
if not OSISAbbrev:
logging.warning(
"toOpenSong: Can't write {} OpenSong book because no OSIS code available"
.format(BBB))
unhandledBooks.append(BBB)
return
writerObject.writeLineOpen('b', ('n', bkData.getAssumedBookNames()[0]))
haveOpenChapter, startedFlag, gotVP, accumulator = False, False, None, ""
C, V = '-1', '-1' # So first/id line starts at -1:0
for processedBibleEntry in bkData._processedLines: # Process internal Bible data lines
marker, text, extras = processedBibleEntry.getMarker(
), processedBibleEntry.getCleanText(
), processedBibleEntry.getExtras()
#dPrint( 'Quiet', debuggingThisModule, marker, repr(text) )
#if text: assert text[0] != ' '
if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS:
continue # Just ignore added markers -- not needed here
if marker in USFM_PRECHAPTER_MARKERS:
if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag:
assert C == '-1' or marker == 'rem' or marker.startswith(
'mte')
V = str(int(V) + 1)
if marker in OFTEN_IGNORED_USFM_HEADER_MARKERS or marker in (
'ie', ): # Just ignore these lines
ignoredMarkers.add(marker)
elif marker == 'c':
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
accumulator = ''
if haveOpenChapter:
writerObject.writeLineClose('c')
C, V = text, '0'
writerObject.writeLineOpen('c', ('n', text))
haveOpenChapter = True
elif marker in (
'c#',
): # These are the markers that we can safely ignore for this export
ignoredMarkers.add(marker)
elif marker == 'vp#': # This precedes a v field and has the verse number to be printed
gotVP = text # Just remember it for now
elif marker == 'v':
V = text
if gotVP: # this is the verse number to be published
text = gotVP
gotVP = None
startedFlag = True
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
accumulator = ''
#dPrint( 'Quiet', debuggingThisModule, "Text {!r}".format( text ) )
if not text:
logging.warning("createOpenSongXML: Missing text for v")
continue
verseNumberString = text.replace('<', '').replace(
'>', ''
).replace(
'"', ''
) # Used below but remove anything that'll cause a big XML problem later
elif marker in ('mt1','mt2','mt3','mt4', 'mte1','mte2','mte3','mte4', 'ms1','ms2','ms3','ms4', ) \
or marker in USFM_ALL_INTRODUCTION_MARKERS \
or marker in ('s1','s2','s3','s4', 'r','sr','mr', 'd','sp','cd', 'cl','lit', ):
ignoredMarkers.add(marker)
elif marker in USFM_BIBLE_PARAGRAPH_MARKERS:
if BibleOrgSysGlobals.debugFlag: assert not text and not extras
ignoredMarkers.add(marker)
elif marker in (
'b',
'nb',
'ib',
):
if BibleOrgSysGlobals.debugFlag: assert not text and not extras
ignoredMarkers.add(marker)
elif marker in (
'v~',
'p~',
):
if BibleOrgSysGlobals.debugFlag: assert text or extras
if not text: # this is an empty (untranslated) verse
text = '- - -' # but we'll put in a filler
if startedFlag:
accumulator += (' ' if accumulator else
'') + BibleOrgSysGlobals.makeSafeXML(text)
else:
if text:
logging.warning(
"toOpenSong: lost text in {} field in {} {}:{} {!r}".
format(marker, BBB, C, V, text))
#if BibleOrgSysGlobals.debugFlag: halt
if extras:
logging.warning(
"toOpenSong: lost extras in {} field in {} {}:{}".
format(marker, BBB, C, V))
#if BibleOrgSysGlobals.debugFlag: halt
unhandledMarkers.add(marker)
if extras and marker not in (
'v~',
'p~',
) and marker not in ignoredMarkers:
logging.critical(
"toOpenSong: extras not handled for {} at {} {}:{}".format(
marker, BBB, C, V))
if accumulator:
writerObject.writeLineOpenClose('v', accumulator,
('n', verseNumberString))
if haveOpenChapter:
writerObject.writeLineClose('c')
writerObject.writeLineClose('b')
# end of createOpenSongXML.writeOpenSongBook
# Set-up our Bible reference system
if 'PublicationCode' not in controlDict or controlDict[
'PublicationCode'] == 'GENERIC':
BOS = BibleObject.genericBOS
BRL = BibleObject.genericBRL
else:
BOS = BibleOrganisationalSystem(controlDict['PublicationCode'])
BRL = BibleReferenceList(BOS, BibleObject=None)
vPrint('Info', debuggingThisModule, _(" Exporting to OpenSong format…"))
try:
osOFn = controlDict['OpenSongOutputFilename']
except KeyError:
osOFn = 'Bible.osong'
filename = BibleOrgSysGlobals.makeSafeFilename(osOFn)
xw = MLWriter(filename, outputFolder)
xw.setHumanReadable()
xw.start()
xw.writeLineOpen('Bible')
for BBB, bookData in BibleObject.books.items():
writeOpenSongBook(xw, BBB, bookData)
xw.writeLineClose('Bible')
xw.close()
if ignoredMarkers:
logging.info("createOpenSongXML: Ignored markers were {}".format(
ignoredMarkers))
vPrint(
'Info', debuggingThisModule,
" " + _("WARNING: Ignored createOpenSongXML markers were {}").format(
ignoredMarkers))
if unhandledMarkers:
logging.warning("createOpenSongXML: Unhandled markers were {}".format(
unhandledMarkers))
vPrint(
'Normal', debuggingThisModule,
" " + _("WARNING: Unhandled toOpenSong markers were {}").format(
unhandledMarkers))
if unhandledBooks:
logging.warning("createOpenSongXML: Unhandled books were {}".format(
unhandledBooks))
vPrint(
'Normal', debuggingThisModule, " " +
_("WARNING: Unhandled createOpenSongXML books were {}").format(
unhandledBooks))
# Now create a zipped version
filepath = os.path.join(outputFolder, filename)
vPrint('Info', debuggingThisModule,
" Zipping {} OpenSong file…".format(filename))
zf = zipfile.ZipFile(filepath + '.zip',
'w',
compression=zipfile.ZIP_DEFLATED)
zf.write(filepath, filename)
zf.close()
if validationSchema: return xw.validate(validationSchema)
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
vPrint('Quiet', debuggingThisModule,
" createOpenSongXML finished successfully.")
return True
def createMySwordModule(self, outputFolder, controlDict):
"""
Create a SQLite3 database module for the program MySword.
self here is a Bible object with _processedLines
"""
import tarfile
from BibleOrgSys.Internals.InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS
from BibleOrgSys.Formats.theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine
def writeMSBook(sqlObject, BBB: str, ourGlobals):
"""
Writes a book to the MySword sqlObject file.
"""
nonlocal lineCount
bkData = self.books[BBB] if BBB in self.books else None
#dPrint( 'Quiet', debuggingThisModule, bkData._processedLines )
verseList = BOS.getNumVersesList(BBB)
nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB)
numC, numV = len(verseList), verseList[0]
ourGlobals['line'], ourGlobals['lastLine'] = '', None
ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals[
'pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals[
'pi7'] = False
if bkData:
# Write book headings (stuff before chapter 1)
ourGlobals['line'] = theWordHandleIntroduction(
BBB, bkData, ourGlobals)
# Write the verses
C = V = 1
ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None
while True:
verseData = None
if bkData:
try:
result = bkData.getContextVerseData((
BBB,
str(C),
str(V),
))
verseData, context = result
except KeyError: # Missing verses
logging.warning(
"BibleWriter.createMySwordModule: missing source verse at {} {}:{}"
.format(BBB, C, V))
# Handle some common versification anomalies
if (BBB, C, V) == ('JN3', 1,
14): # Add text for v15 if it exists
try:
result15 = bkData.getContextVerseData((
'JN3',
'1',
'15',
))
verseData15, context15 = result15
verseData.extend(verseData15)
except KeyError:
pass # just ignore it
elif (BBB, C, V) == ('REV', 12,
17): # Add text for v15 if it exists
try:
result18 = bkData.getContextVerseData((
'REV',
'12',
'18',
))
verseData18, context18 = result18
verseData.extend(verseData18)
except KeyError:
pass # just ignore it
composedLine = ''
if verseData:
composedLine = theWordComposeVerseLine(
BBB, C, V, verseData, ourGlobals)
# Stay one line behind (because paragraph indicators get appended to the previous line)
if ourGlobals['lastBCV'] is not None \
and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
ourGlobals['lastLine'] = composedLine
ourGlobals['lastBCV'] = (nBBB, C, V)
V += 1
if V > numV:
C += 1
if C > numC:
break
else: # next chapter only
numV = verseList[C - 1]
V = 1
#assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything
# Write the last line of the file
if ourGlobals[
'lastLine']: # don't bother writing blank (unfinished?) verses
sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \
(ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) )
lineCount += 1
# end of createMySwordModule.writeMSBook
# Set-up their Bible reference system
BOS = BibleOrganisationalSystem('GENERIC-KJV-66-ENG')
#BRL = BibleReferenceList( BOS, BibleObject=None )
# Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.)
if len(self) <= (39 + 4) and self.containsAnyOT39Books(
) and not self.containsAnyNT27Books():
testament, startBBB, endBBB = 'OT', 'GEN', 'MAL'
booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines
elif len(self) <= (27 + 4) and self.containsAnyNT27Books(
) and not self.containsAnyOT39Books():
testament, startBBB, endBBB = 'NT', 'MAT', 'REV'
booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines
else: # assume it's an entire Bible
testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV'
booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines
extension = '.bbl.mybible'
vPrint('Info', debuggingThisModule, _(" Exporting to MySword format…"))
mySettings = {}
mySettings['unhandledMarkers'] = set()
handledBooks = []
if 'MySwordOutputFilename' in controlDict:
filename = controlDict['MySwordOutputFilename']
elif self.sourceFilename:
filename = self.sourceFilename
elif self.shortName:
filename = self.shortName
elif self.abbreviation:
filename = self.abbreviation
elif self.name:
filename = self.name
else:
filename = 'export'
if not filename.endswith(extension):
filename += extension # Make sure that we have the right file extension
filepath = os.path.join(outputFolder,
BibleOrgSysGlobals.makeSafeFilename(filename))
if os.path.exists(filepath): os.remove(filepath)
vPrint('Info', debuggingThisModule,
' createMySwordModule: ' + _("Writing {!r}…").format(filepath))
conn = sqlite3.connect(filepath)
cursor = conn.cursor()
# First write the settings Details table
exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete
customCSS = self.getSetting('CustomCSS')
if customCSS: exeStr += ', CustomCSS TEXT'
exeStr += ')'
cursor.execute(exeStr)
values = []
description = self.getSetting('Description')
if not description: description = self.getSetting('description')
if not description: description = self.name
values.append(description)
if self.abbreviation: abbreviation = self.abbreviation
else: abbreviation = self.getSetting('WorkAbbreviation')
if not abbreviation: abbreviation = self.name[:3].upper()
values.append(abbreviation)
comments = self.getSetting('Comments')
values.append(comments)
version = self.getSetting('Version')
values.append(version)
versionDate = self.getSetting('VersionDate')
values.append(versionDate)
publishDate = self.getSetting('PublishDate')
values.append(publishDate)
rightToLeft = self.getSetting('RightToLeft')
values.append(rightToLeft)
values.append(True if testament == 'OT' or testament == 'BOTH' else False)
values.append(True if testament == 'NT' or testament == 'BOTH' else False)
Strong = self.getSetting('Strong')
values.append(Strong if Strong else False)
if customCSS: values.append(customCSS)
exeStr = 'INSERT INTO "Details" VALUES(' + '?,' * (len(values) - 1) + '?)'
#dPrint( 'Quiet', debuggingThisModule, exeStr, values )
cursor.execute(exeStr, values)
#if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values )
#else: # Not debugging
#try: cursor.execute( exeStr, values )
#except sqlite3.InterfaceError:
#logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) )
# Now create and fill the Bible table
cursor.execute(
'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))'
)
conn.commit() # save (commit) the changes
BBB, lineCount = startBBB, 0
while True: # Write each Bible book in the KJV order
writeMSBook(cursor, BBB, mySettings)
conn.commit() # save (commit) the changes
handledBooks.append(BBB)
if BBB == endBBB: break
BBB = BOS.getNextBookCode(BBB)
conn.commit() # save (commit) the changes
cursor.close()
if mySettings['unhandledMarkers']:
logging.warning(
"BibleWriter.createMySwordModule: Unhandled markers were {}".
format(mySettings['unhandledMarkers']))
vPrint(
'Normal', debuggingThisModule, " " +
_("WARNING: Unhandled createMySwordModule markers were {}").format(
mySettings['unhandledMarkers']))
unhandledBooks = []
for BBB in self.getBookList():
if BBB not in handledBooks: unhandledBooks.append(BBB)
if unhandledBooks:
logging.warning("createMySwordModule: Unhandled books were {}".format(
unhandledBooks))
vPrint(
'Normal', debuggingThisModule, " " +
_("WARNING: Unhandled createMySwordModule books were {}").format(
unhandledBooks))
# Now create the gzipped file
vPrint('Info', debuggingThisModule,
" Compressing {} MySword file…".format(filename))
tar = tarfile.open(filepath + '.gz', 'w:gz')
tar.add(filepath)
tar.close()
if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1:
vPrint('Quiet', debuggingThisModule,
" BibleWriter.createMySwordModule finished successfully.")
return True
def __init__(self,
parameterOne,
resourcesObject=None,
downloadAllBooks=False) -> None:
"""
Create the Door43 cataloged Bible object.
parameterOne can be:
a catalog dictionary entry (and second parameter must be None)
or
an index into the BibleList in the resourcesObject passed as the second parameter
"""
fnPrint(
debuggingThisModule,
f"DCSBible.__init__( {parameterOne}, {resourcesObject}, {downloadAllBooks} )…"
)
if isinstance(parameterOne, dict):
assert resourcesObject is None
resourceDict = parameterOne
else:
assert isinstance(parameterOne, int)
assert resourcesObject # why ??? and isinstance( resourcesObject, Door43CatalogResources )
resourceDict = resourcesObject.getBibleResourceDict(parameterOne)
assert resourceDict and isinstance(resourceDict, dict)
#dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict )
#dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict.keys() )
self.baseURL = resourceDict['html_url']
#dPrint( 'Quiet', debuggingThisModule, 'self.baseURL', self.baseURL )
adjustedRepoName = resourceDict['full_name'].replace('/', '--')
#dPrint( 'Quiet', debuggingThisModule, 'adjustedRepoName', adjustedRepoName )
desiredFolderName = BibleOrgSysGlobals.makeSafeFilename(
adjustedRepoName)
unzippedFolderpath = DEFAULT_DOWNLOAD_FOLDERPATH.joinpath(
f'{adjustedRepoName}/')
if downloadAllBooks:
# See if files already exist and are current (so don't download again)
alreadyDownloadedFlag = False
if os.path.isdir(unzippedFolderpath):
#dPrint( 'Quiet', debuggingThisModule, f"Issued: {resourceDict['issued']}" )
updatedDatetime = datetime.strptime(resourceDict['updated_at'],
'%Y-%m-%dT%H:%M:%SZ')
#dPrint( 'Quiet', debuggingThisModule, f"updatedDatetime: {updatedDatetime}" )
#dPrint( 'Quiet', debuggingThisModule, f"folder: {os.stat(unzippedFolderpath).st_mtime}" )
folderModifiedDatetime = datetime.fromtimestamp(
os.stat(unzippedFolderpath).st_mtime)
#dPrint( 'Quiet', debuggingThisModule, f"folderModifiedDatetime: {folderModifiedDatetime}" )
alreadyDownloadedFlag = folderModifiedDatetime > updatedDatetime
#dPrint( 'Quiet', debuggingThisModule, f"alreadyDownloadedFlag: {alreadyDownloadedFlag}" )
if alreadyDownloadedFlag:
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint(
'Quiet', debuggingThisModule,
"Skipping download because folder '{}' already exists."
.format(unzippedFolderpath))
else: # Download the zip file (containing all the USFM files, README.md, LICENSE.md, manifest.yaml, etc.)
# TODO: Change to .tar.gz instead of zip
zipURL = self.baseURL + '/archive/master.zip' # '/archive/master.tar.gz'
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint('Quiet', debuggingThisModule,
"Downloading entire repo from '{}'…".format(zipURL))
try:
HTTPResponseObject = urllib.request.urlopen(zipURL)
except urllib.error.URLError as err:
#errorClass, exceptionInstance, traceback = sys.exc_info()
#dPrint( 'Quiet', debuggingThisModule, '{!r} {!r} {!r}'.format( errorClass, exceptionInstance, traceback ) )
logging.critical("DCS URLError '{}' from {}".format(
err, zipURL))
return
#dPrint( 'Quiet', debuggingThisModule, " HTTPResponseObject", HTTPResponseObject )
contentType = HTTPResponseObject.info().get('content-type')
if BibleOrgSysGlobals.debugFlag and debuggingThisModule:
vPrint('Quiet', debuggingThisModule, " contentType",
repr(contentType))
if contentType == 'application/octet-stream':
try:
os.makedirs(unzippedFolderpath)
except FileExistsError:
pass
downloadedData = HTTPResponseObject.read()
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint(
'Quiet', debuggingThisModule,
f" Downloaded {len(downloadedData):,} bytes from '{zipURL}'"
)
# Bug in Python up to 3.7 makes this not work for large aligned Bibles (3+ MB)
# myTempFile = tempfile.SpooledTemporaryFile()
myTempFile = tempfile.TemporaryFile()
myTempFile.write(downloadedData)
with zipfile.ZipFile(myTempFile) as myzip:
# NOTE: Could be a security risk here
myzip.extractall(unzippedFolderpath)
myTempFile.close() # Automatically deletes the file
else:
vPrint('Quiet', debuggingThisModule, " contentType",
repr(contentType))
halt # unknown content type
self.downloadedAllBooks = True
# There's probably a folder inside this folder
folders = os.listdir(unzippedFolderpath)
#dPrint( 'Quiet', debuggingThisModule, 'folders', folders )
assert len(
folders
) == 1 # else maybe a previous download failed -- just manually delete the folder
desiredFolderName = folders[0] + '/'
#dPrint( 'Quiet', debuggingThisModule, 'desiredFolderName', desiredFolderName )
USFMBible.__init__(self,
os.path.join(unzippedFolderpath,
desiredFolderName),
givenName=resourceDict['name'])
else: # didn't request all books to be downloaded at once
self.downloadedAllBooks = False
self.attemptedDownload = {}
try:
os.makedirs(unzippedFolderpath)
except FileExistsError:
pass
USFMBible.__init__(self,
unzippedFolderpath,
givenName=resourceDict['name'])
self.objectNameString = 'DCS USFM Bible object'
self.uWencoded = True