def fullDemo() -> None: """ Full demo to check class is working """ BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion, LAST_MODIFIED_DATE) testFolderpath = Path('/mnt/SSDs/Bibles/unfoldingWordHelps/en_obs/') if 1: # demo the file checking code -- first with the whole folder and then with only one folder vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA1") result1 = uWOBSBibleFileCheck(testFolderpath) vPrint('Normal', debuggingThisModule, "uW OBS TestA1", result1) vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA2") result2 = uWOBSBibleFileCheck( testFolderpath, autoLoad=True) # But doesn't preload books vPrint('Normal', debuggingThisModule, "uW OBS TestA2", result2) #result2.loadMetadataFile( os.path.join( testFolderpath, "BooknamesMetadata.txt" ) ) if BibleOrgSysGlobals.strictCheckingFlag: result2.check() #dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result2.getCheckResults() #dPrint( 'Quiet', debuggingThisModule, bibleErrors ) #if BibleOrgSysGlobals.commandLineArguments.export: ###result2.toDrupalBible() #result2.doAllExports( wantPhotoBible=False, wantODFs=False, wantPDFs=False ) vPrint('Quiet', debuggingThisModule, "\nuW OBS TestA3") result3 = uWOBSBibleFileCheck(testFolderpath, autoLoad=True, autoLoadBooks=True) vPrint('Normal', debuggingThisModule, "uW OBS TestA3", result3) #result3.loadMetadataFile( os.path.join( testFolderpath, "BooknamesMetadata.txt" ) ) for BBB in ('OBS', 'RUT', 'JN3'): vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gCVD", result3.getContextVerseData(('OBS', '1', '1', ''))) vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gVDL", result3.getVerseDataList(('OBS', '1', '1', ''))) vPrint('Quiet', debuggingThisModule, f"OBS 1:1 gVT", result3.getVerseText(('OBS', '1', '1', ''))) if BibleOrgSysGlobals.strictCheckingFlag: result3.check() #dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] ) bibleErrors = result3.getCheckResults() #dPrint( 'Quiet', debuggingThisModule, bibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##result3.toDrupalBible() result3.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) if 0: # all discovered modules in the test folder foundFolders, foundFiles = [], [] for something in os.listdir(testFolderpath): somepath = os.path.join(testFolderpath, something) if os.path.isdir(somepath): foundFolders.append(something) elif os.path.isfile(somepath): foundFiles.append(something) if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work vPrint( 'Normal', debuggingThisModule, "\nTrying all {} discovered modules…".format( len(foundFolders))) parameters = [folderName for folderName in sorted(foundFolders)] BibleOrgSysGlobals.alreadyMultiprocessing = True with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses ) as pool: # start worker processes results = pool.map(testBCV, parameters) # have the pool do our loads assert len(results) == len( parameters ) # Results (all None) are actually irrelevant to us here BibleOrgSysGlobals.alreadyMultiprocessing = False else: # Just single threaded for j, someFolder in enumerate(sorted(foundFolders)): vPrint('Normal', debuggingThisModule, "\nuW OBS D{}/ Trying {}".format(j + 1, someFolder)) #myTestFolder = os.path.join( testFolderpath, someFolder+'/' ) testBCV(someFolder) if 0: # Load and process some of our test versions count = 0 for name, encoding, testFolder in ( ("Matigsalug", 'utf-8', BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('BCVTest1/') ), ("Matigsalug", 'utf-8', BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('BCVTest2/') ), ("Exported", 'utf-8', "Tests/BOS_BCV_Export/"), ): count += 1 if os.access(testFolder, os.R_OK): vPrint('Quiet', debuggingThisModule, "\nuW OBS A{}/".format(count)) uWnB = uWOBSBible(testFolder, name, encoding=encoding) uWnB.load() if BibleOrgSysGlobals.verbosityLevel > 1: vPrint('Quiet', debuggingThisModule, "Gen assumed book name:", repr(uWnB.getAssumedBookName('GEN'))) vPrint('Quiet', debuggingThisModule, "Gen long TOC book name:", repr(uWnB.getLongTOCName('GEN'))) vPrint('Quiet', debuggingThisModule, "Gen short TOC book name:", repr(uWnB.getShortTOCName('GEN'))) vPrint('Quiet', debuggingThisModule, "Gen book abbreviation:", repr(uWnB.getBooknameAbbreviation('GEN'))) vPrint('Quiet', debuggingThisModule, uWnB) if BibleOrgSysGlobals.strictCheckingFlag: uWnB.check() #dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] ) bcbibleErrors = uWnB.getCheckResults() #dPrint( 'Quiet', debuggingThisModule, bcbibleErrors ) if BibleOrgSysGlobals.commandLineArguments.export: ##uWnB.toDrupalBible() uWnB.doAllExports(wantPhotoBible=False, wantODFs=False, wantPDFs=False) newObj = BibleOrgSysGlobals.unpickleObject( BibleOrgSysGlobals.makeSafeFilename(name) + '.pickle', os.path.join("BOSOutputFiles/", "BOS_Bible_Object_Pickle/")) vPrint('Quiet', debuggingThisModule, "newObj is", newObj) else: vPrint( 'Quiet', debuggingThisModule, f"\nSorry, test folder '{testFolder}' is not readable on this computer." )
def createOpenSongXML(BibleObject, outputFolder=None, controlDict=None, validationSchema=None): """ Using settings from the given control file, converts the USFM information to a UTF-8 OpenSong XML file. This format is roughly documented at http://de.wikipedia.org/wiki/OpenSong_XML but more fields can be discovered by looking at downloaded files. """ vPrint('Normal', debuggingThisModule, "Running createOpenSongXML…") if BibleOrgSysGlobals.debugFlag: assert BibleObject.books ignoredMarkers, unhandledMarkers, unhandledBooks = set(), set(), [] def writeOpenSongBook(writerObject, BBB: str, bkData): """Writes a book to the OpenSong XML writerObject.""" #dPrint( 'Quiet', debuggingThisModule, 'BIBLEBOOK', [('bnumber',BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB)), ('bname',BibleOrgSysGlobals.loadedBibleBooksCodes.getEnglishName_NR(BBB)), ('bsname',BibleOrgSysGlobals.loadedBibleBooksCodes.getOSISAbbreviation(BBB))] ) OSISAbbrev = BibleOrgSysGlobals.loadedBibleBooksCodes.getOSISAbbreviation( BBB) if not OSISAbbrev: logging.warning( "toOpenSong: Can't write {} OpenSong book because no OSIS code available" .format(BBB)) unhandledBooks.append(BBB) return writerObject.writeLineOpen('b', ('n', bkData.getAssumedBookNames()[0])) haveOpenChapter, startedFlag, gotVP, accumulator = False, False, None, "" C, V = '-1', '-1' # So first/id line starts at -1:0 for processedBibleEntry in bkData._processedLines: # Process internal Bible data lines marker, text, extras = processedBibleEntry.getMarker( ), processedBibleEntry.getCleanText( ), processedBibleEntry.getExtras() #dPrint( 'Quiet', debuggingThisModule, marker, repr(text) ) #if text: assert text[0] != ' ' if '¬' in marker or marker in BOS_ADDED_NESTING_MARKERS: continue # Just ignore added markers -- not needed here if marker in USFM_PRECHAPTER_MARKERS: if debuggingThisModule or BibleOrgSysGlobals.debugFlag or BibleOrgSysGlobals.strictCheckingFlag: assert C == '-1' or marker == 'rem' or marker.startswith( 'mte') V = str(int(V) + 1) if marker in OFTEN_IGNORED_USFM_HEADER_MARKERS or marker in ( 'ie', ): # Just ignore these lines ignoredMarkers.add(marker) elif marker == 'c': if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) accumulator = '' if haveOpenChapter: writerObject.writeLineClose('c') C, V = text, '0' writerObject.writeLineOpen('c', ('n', text)) haveOpenChapter = True elif marker in ( 'c#', ): # These are the markers that we can safely ignore for this export ignoredMarkers.add(marker) elif marker == 'vp#': # This precedes a v field and has the verse number to be printed gotVP = text # Just remember it for now elif marker == 'v': V = text if gotVP: # this is the verse number to be published text = gotVP gotVP = None startedFlag = True if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) accumulator = '' #dPrint( 'Quiet', debuggingThisModule, "Text {!r}".format( text ) ) if not text: logging.warning("createOpenSongXML: Missing text for v") continue verseNumberString = text.replace('<', '').replace( '>', '' ).replace( '"', '' ) # Used below but remove anything that'll cause a big XML problem later elif marker in ('mt1','mt2','mt3','mt4', 'mte1','mte2','mte3','mte4', 'ms1','ms2','ms3','ms4', ) \ or marker in USFM_ALL_INTRODUCTION_MARKERS \ or marker in ('s1','s2','s3','s4', 'r','sr','mr', 'd','sp','cd', 'cl','lit', ): ignoredMarkers.add(marker) elif marker in USFM_BIBLE_PARAGRAPH_MARKERS: if BibleOrgSysGlobals.debugFlag: assert not text and not extras ignoredMarkers.add(marker) elif marker in ( 'b', 'nb', 'ib', ): if BibleOrgSysGlobals.debugFlag: assert not text and not extras ignoredMarkers.add(marker) elif marker in ( 'v~', 'p~', ): if BibleOrgSysGlobals.debugFlag: assert text or extras if not text: # this is an empty (untranslated) verse text = '- - -' # but we'll put in a filler if startedFlag: accumulator += (' ' if accumulator else '') + BibleOrgSysGlobals.makeSafeXML(text) else: if text: logging.warning( "toOpenSong: lost text in {} field in {} {}:{} {!r}". format(marker, BBB, C, V, text)) #if BibleOrgSysGlobals.debugFlag: halt if extras: logging.warning( "toOpenSong: lost extras in {} field in {} {}:{}". format(marker, BBB, C, V)) #if BibleOrgSysGlobals.debugFlag: halt unhandledMarkers.add(marker) if extras and marker not in ( 'v~', 'p~', ) and marker not in ignoredMarkers: logging.critical( "toOpenSong: extras not handled for {} at {} {}:{}".format( marker, BBB, C, V)) if accumulator: writerObject.writeLineOpenClose('v', accumulator, ('n', verseNumberString)) if haveOpenChapter: writerObject.writeLineClose('c') writerObject.writeLineClose('b') # end of createOpenSongXML.writeOpenSongBook # Set-up our Bible reference system if 'PublicationCode' not in controlDict or controlDict[ 'PublicationCode'] == 'GENERIC': BOS = BibleObject.genericBOS BRL = BibleObject.genericBRL else: BOS = BibleOrganisationalSystem(controlDict['PublicationCode']) BRL = BibleReferenceList(BOS, BibleObject=None) vPrint('Info', debuggingThisModule, _(" Exporting to OpenSong format…")) try: osOFn = controlDict['OpenSongOutputFilename'] except KeyError: osOFn = 'Bible.osong' filename = BibleOrgSysGlobals.makeSafeFilename(osOFn) xw = MLWriter(filename, outputFolder) xw.setHumanReadable() xw.start() xw.writeLineOpen('Bible') for BBB, bookData in BibleObject.books.items(): writeOpenSongBook(xw, BBB, bookData) xw.writeLineClose('Bible') xw.close() if ignoredMarkers: logging.info("createOpenSongXML: Ignored markers were {}".format( ignoredMarkers)) vPrint( 'Info', debuggingThisModule, " " + _("WARNING: Ignored createOpenSongXML markers were {}").format( ignoredMarkers)) if unhandledMarkers: logging.warning("createOpenSongXML: Unhandled markers were {}".format( unhandledMarkers)) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled toOpenSong markers were {}").format( unhandledMarkers)) if unhandledBooks: logging.warning("createOpenSongXML: Unhandled books were {}".format( unhandledBooks)) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled createOpenSongXML books were {}").format( unhandledBooks)) # Now create a zipped version filepath = os.path.join(outputFolder, filename) vPrint('Info', debuggingThisModule, " Zipping {} OpenSong file…".format(filename)) zf = zipfile.ZipFile(filepath + '.zip', 'w', compression=zipfile.ZIP_DEFLATED) zf.write(filepath, filename) zf.close() if validationSchema: return xw.validate(validationSchema) if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: vPrint('Quiet', debuggingThisModule, " createOpenSongXML finished successfully.") return True
def createMySwordModule(self, outputFolder, controlDict): """ Create a SQLite3 database module for the program MySword. self here is a Bible object with _processedLines """ import tarfile from BibleOrgSys.Internals.InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS from BibleOrgSys.Formats.theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine def writeMSBook(sqlObject, BBB: str, ourGlobals): """ Writes a book to the MySword sqlObject file. """ nonlocal lineCount bkData = self.books[BBB] if BBB in self.books else None #dPrint( 'Quiet', debuggingThisModule, bkData._processedLines ) verseList = BOS.getNumVersesList(BBB) nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB) numC, numV = len(verseList), verseList[0] ourGlobals['line'], ourGlobals['lastLine'] = '', None ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals[ 'pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals[ 'pi7'] = False if bkData: # Write book headings (stuff before chapter 1) ourGlobals['line'] = theWordHandleIntroduction( BBB, bkData, ourGlobals) # Write the verses C = V = 1 ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None while True: verseData = None if bkData: try: result = bkData.getContextVerseData(( BBB, str(C), str(V), )) verseData, context = result except KeyError: # Missing verses logging.warning( "BibleWriter.createMySwordModule: missing source verse at {} {}:{}" .format(BBB, C, V)) # Handle some common versification anomalies if (BBB, C, V) == ('JN3', 1, 14): # Add text for v15 if it exists try: result15 = bkData.getContextVerseData(( 'JN3', '1', '15', )) verseData15, context15 = result15 verseData.extend(verseData15) except KeyError: pass # just ignore it elif (BBB, C, V) == ('REV', 12, 17): # Add text for v15 if it exists try: result18 = bkData.getContextVerseData(( 'REV', '12', '18', )) verseData18, context18 = result18 verseData.extend(verseData18) except KeyError: pass # just ignore it composedLine = '' if verseData: composedLine = theWordComposeVerseLine( BBB, C, V, verseData, ourGlobals) # Stay one line behind (because paragraph indicators get appended to the previous line) if ourGlobals['lastBCV'] is not None \ and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 ourGlobals['lastLine'] = composedLine ourGlobals['lastBCV'] = (nBBB, C, V) V += 1 if V > numV: C += 1 if C > numC: break else: # next chapter only numV = verseList[C - 1] V = 1 #assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything # Write the last line of the file if ourGlobals[ 'lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 # end of createMySwordModule.writeMSBook # Set-up their Bible reference system BOS = BibleOrganisationalSystem('GENERIC-KJV-66-ENG') #BRL = BibleReferenceList( BOS, BibleObject=None ) # Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.) if len(self) <= (39 + 4) and self.containsAnyOT39Books( ) and not self.containsAnyNT27Books(): testament, startBBB, endBBB = 'OT', 'GEN', 'MAL' booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines elif len(self) <= (27 + 4) and self.containsAnyNT27Books( ) and not self.containsAnyOT39Books(): testament, startBBB, endBBB = 'NT', 'MAT', 'REV' booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines else: # assume it's an entire Bible testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV' booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines extension = '.bbl.mybible' vPrint('Info', debuggingThisModule, _(" Exporting to MySword format…")) mySettings = {} mySettings['unhandledMarkers'] = set() handledBooks = [] if 'MySwordOutputFilename' in controlDict: filename = controlDict['MySwordOutputFilename'] elif self.sourceFilename: filename = self.sourceFilename elif self.shortName: filename = self.shortName elif self.abbreviation: filename = self.abbreviation elif self.name: filename = self.name else: filename = 'export' if not filename.endswith(extension): filename += extension # Make sure that we have the right file extension filepath = os.path.join(outputFolder, BibleOrgSysGlobals.makeSafeFilename(filename)) if os.path.exists(filepath): os.remove(filepath) vPrint('Info', debuggingThisModule, ' createMySwordModule: ' + _("Writing {!r}…").format(filepath)) conn = sqlite3.connect(filepath) cursor = conn.cursor() # First write the settings Details table exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete customCSS = self.getSetting('CustomCSS') if customCSS: exeStr += ', CustomCSS TEXT' exeStr += ')' cursor.execute(exeStr) values = [] description = self.getSetting('Description') if not description: description = self.getSetting('description') if not description: description = self.name values.append(description) if self.abbreviation: abbreviation = self.abbreviation else: abbreviation = self.getSetting('WorkAbbreviation') if not abbreviation: abbreviation = self.name[:3].upper() values.append(abbreviation) comments = self.getSetting('Comments') values.append(comments) version = self.getSetting('Version') values.append(version) versionDate = self.getSetting('VersionDate') values.append(versionDate) publishDate = self.getSetting('PublishDate') values.append(publishDate) rightToLeft = self.getSetting('RightToLeft') values.append(rightToLeft) values.append(True if testament == 'OT' or testament == 'BOTH' else False) values.append(True if testament == 'NT' or testament == 'BOTH' else False) Strong = self.getSetting('Strong') values.append(Strong if Strong else False) if customCSS: values.append(customCSS) exeStr = 'INSERT INTO "Details" VALUES(' + '?,' * (len(values) - 1) + '?)' #dPrint( 'Quiet', debuggingThisModule, exeStr, values ) cursor.execute(exeStr, values) #if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values ) #else: # Not debugging #try: cursor.execute( exeStr, values ) #except sqlite3.InterfaceError: #logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) ) # Now create and fill the Bible table cursor.execute( 'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))' ) conn.commit() # save (commit) the changes BBB, lineCount = startBBB, 0 while True: # Write each Bible book in the KJV order writeMSBook(cursor, BBB, mySettings) conn.commit() # save (commit) the changes handledBooks.append(BBB) if BBB == endBBB: break BBB = BOS.getNextBookCode(BBB) conn.commit() # save (commit) the changes cursor.close() if mySettings['unhandledMarkers']: logging.warning( "BibleWriter.createMySwordModule: Unhandled markers were {}". format(mySettings['unhandledMarkers'])) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled createMySwordModule markers were {}").format( mySettings['unhandledMarkers'])) unhandledBooks = [] for BBB in self.getBookList(): if BBB not in handledBooks: unhandledBooks.append(BBB) if unhandledBooks: logging.warning("createMySwordModule: Unhandled books were {}".format( unhandledBooks)) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled createMySwordModule books were {}").format( unhandledBooks)) # Now create the gzipped file vPrint('Info', debuggingThisModule, " Compressing {} MySword file…".format(filename)) tar = tarfile.open(filepath + '.gz', 'w:gz') tar.add(filepath) tar.close() if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: vPrint('Quiet', debuggingThisModule, " BibleWriter.createMySwordModule finished successfully.") return True
def __init__(self, parameterOne, resourcesObject=None, downloadAllBooks=False) -> None: """ Create the Door43 cataloged Bible object. parameterOne can be: a catalog dictionary entry (and second parameter must be None) or an index into the BibleList in the resourcesObject passed as the second parameter """ fnPrint( debuggingThisModule, f"DCSBible.__init__( {parameterOne}, {resourcesObject}, {downloadAllBooks} )…" ) if isinstance(parameterOne, dict): assert resourcesObject is None resourceDict = parameterOne else: assert isinstance(parameterOne, int) assert resourcesObject # why ??? and isinstance( resourcesObject, Door43CatalogResources ) resourceDict = resourcesObject.getBibleResourceDict(parameterOne) assert resourceDict and isinstance(resourceDict, dict) #dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict ) #dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict.keys() ) self.baseURL = resourceDict['html_url'] #dPrint( 'Quiet', debuggingThisModule, 'self.baseURL', self.baseURL ) adjustedRepoName = resourceDict['full_name'].replace('/', '--') #dPrint( 'Quiet', debuggingThisModule, 'adjustedRepoName', adjustedRepoName ) desiredFolderName = BibleOrgSysGlobals.makeSafeFilename( adjustedRepoName) unzippedFolderpath = DEFAULT_DOWNLOAD_FOLDERPATH.joinpath( f'{adjustedRepoName}/') if downloadAllBooks: # See if files already exist and are current (so don't download again) alreadyDownloadedFlag = False if os.path.isdir(unzippedFolderpath): #dPrint( 'Quiet', debuggingThisModule, f"Issued: {resourceDict['issued']}" ) updatedDatetime = datetime.strptime(resourceDict['updated_at'], '%Y-%m-%dT%H:%M:%SZ') #dPrint( 'Quiet', debuggingThisModule, f"updatedDatetime: {updatedDatetime}" ) #dPrint( 'Quiet', debuggingThisModule, f"folder: {os.stat(unzippedFolderpath).st_mtime}" ) folderModifiedDatetime = datetime.fromtimestamp( os.stat(unzippedFolderpath).st_mtime) #dPrint( 'Quiet', debuggingThisModule, f"folderModifiedDatetime: {folderModifiedDatetime}" ) alreadyDownloadedFlag = folderModifiedDatetime > updatedDatetime #dPrint( 'Quiet', debuggingThisModule, f"alreadyDownloadedFlag: {alreadyDownloadedFlag}" ) if alreadyDownloadedFlag: if BibleOrgSysGlobals.verbosityLevel > 1: vPrint( 'Quiet', debuggingThisModule, "Skipping download because folder '{}' already exists." .format(unzippedFolderpath)) else: # Download the zip file (containing all the USFM files, README.md, LICENSE.md, manifest.yaml, etc.) # TODO: Change to .tar.gz instead of zip zipURL = self.baseURL + '/archive/master.zip' # '/archive/master.tar.gz' if BibleOrgSysGlobals.verbosityLevel > 1: vPrint('Quiet', debuggingThisModule, "Downloading entire repo from '{}'…".format(zipURL)) try: HTTPResponseObject = urllib.request.urlopen(zipURL) except urllib.error.URLError as err: #errorClass, exceptionInstance, traceback = sys.exc_info() #dPrint( 'Quiet', debuggingThisModule, '{!r} {!r} {!r}'.format( errorClass, exceptionInstance, traceback ) ) logging.critical("DCS URLError '{}' from {}".format( err, zipURL)) return #dPrint( 'Quiet', debuggingThisModule, " HTTPResponseObject", HTTPResponseObject ) contentType = HTTPResponseObject.info().get('content-type') if BibleOrgSysGlobals.debugFlag and debuggingThisModule: vPrint('Quiet', debuggingThisModule, " contentType", repr(contentType)) if contentType == 'application/octet-stream': try: os.makedirs(unzippedFolderpath) except FileExistsError: pass downloadedData = HTTPResponseObject.read() if BibleOrgSysGlobals.verbosityLevel > 0: vPrint( 'Quiet', debuggingThisModule, f" Downloaded {len(downloadedData):,} bytes from '{zipURL}'" ) # Bug in Python up to 3.7 makes this not work for large aligned Bibles (3+ MB) # myTempFile = tempfile.SpooledTemporaryFile() myTempFile = tempfile.TemporaryFile() myTempFile.write(downloadedData) with zipfile.ZipFile(myTempFile) as myzip: # NOTE: Could be a security risk here myzip.extractall(unzippedFolderpath) myTempFile.close() # Automatically deletes the file else: vPrint('Quiet', debuggingThisModule, " contentType", repr(contentType)) halt # unknown content type self.downloadedAllBooks = True # There's probably a folder inside this folder folders = os.listdir(unzippedFolderpath) #dPrint( 'Quiet', debuggingThisModule, 'folders', folders ) assert len( folders ) == 1 # else maybe a previous download failed -- just manually delete the folder desiredFolderName = folders[0] + '/' #dPrint( 'Quiet', debuggingThisModule, 'desiredFolderName', desiredFolderName ) USFMBible.__init__(self, os.path.join(unzippedFolderpath, desiredFolderName), givenName=resourceDict['name']) else: # didn't request all books to be downloaded at once self.downloadedAllBooks = False self.attemptedDownload = {} try: os.makedirs(unzippedFolderpath) except FileExistsError: pass USFMBible.__init__(self, unzippedFolderpath, givenName=resourceDict['name']) self.objectNameString = 'DCS USFM Bible object' self.uWencoded = True