def createMySwordModule(self, outputFolder, controlDict): """ Create a SQLite3 database module for the program MySword. self here is a Bible object with _processedLines """ import tarfile from BibleOrgSys.Internals.InternalBibleInternals import BOS_ADDED_NESTING_MARKERS, BOS_NESTING_MARKERS from BibleOrgSys.Formats.theWordBible import theWordOTBookLines, theWordNTBookLines, theWordBookLines, theWordHandleIntroduction, theWordComposeVerseLine def writeMSBook(sqlObject, BBB: str, ourGlobals): """ Writes a book to the MySword sqlObject file. """ nonlocal lineCount bkData = self.books[BBB] if BBB in self.books else None #dPrint( 'Quiet', debuggingThisModule, bkData._processedLines ) verseList = BOS.getNumVersesList(BBB) nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB) numC, numV = len(verseList), verseList[0] ourGlobals['line'], ourGlobals['lastLine'] = '', None ourGlobals['pi1'] = ourGlobals['pi2'] = ourGlobals['pi3'] = ourGlobals[ 'pi4'] = ourGlobals['pi5'] = ourGlobals['pi6'] = ourGlobals[ 'pi7'] = False if bkData: # Write book headings (stuff before chapter 1) ourGlobals['line'] = theWordHandleIntroduction( BBB, bkData, ourGlobals) # Write the verses C = V = 1 ourGlobals['lastLine'] = ourGlobals['lastBCV'] = None while True: verseData = None if bkData: try: result = bkData.getContextVerseData(( BBB, str(C), str(V), )) verseData, context = result except KeyError: # Missing verses logging.warning( "BibleWriter.createMySwordModule: missing source verse at {} {}:{}" .format(BBB, C, V)) # Handle some common versification anomalies if (BBB, C, V) == ('JN3', 1, 14): # Add text for v15 if it exists try: result15 = bkData.getContextVerseData(( 'JN3', '1', '15', )) verseData15, context15 = result15 verseData.extend(verseData15) except KeyError: pass # just ignore it elif (BBB, C, V) == ('REV', 12, 17): # Add text for v15 if it exists try: result18 = bkData.getContextVerseData(( 'REV', '12', '18', )) verseData18, context18 = result18 verseData.extend(verseData18) except KeyError: pass # just ignore it composedLine = '' if verseData: composedLine = theWordComposeVerseLine( BBB, C, V, verseData, ourGlobals) # Stay one line behind (because paragraph indicators get appended to the previous line) if ourGlobals['lastBCV'] is not None \ and ourGlobals['lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 ourGlobals['lastLine'] = composedLine ourGlobals['lastBCV'] = (nBBB, C, V) V += 1 if V > numV: C += 1 if C > numC: break else: # next chapter only numV = verseList[C - 1] V = 1 #assert not ourGlobals['line'] and not ourGlobals['lastLine'] # We should have written everything # Write the last line of the file if ourGlobals[ 'lastLine']: # don't bother writing blank (unfinished?) verses sqlObject.execute( 'INSERT INTO "Bible" VALUES(?,?,?,?)', \ (ourGlobals['lastBCV'][0],ourGlobals['lastBCV'][1],ourGlobals['lastBCV'][2],ourGlobals['lastLine']) ) lineCount += 1 # end of createMySwordModule.writeMSBook # Set-up their Bible reference system BOS = BibleOrganisationalSystem('GENERIC-KJV-66-ENG') #BRL = BibleReferenceList( BOS, BibleObject=None ) # Try to figure out if it's an OT/NT or what (allow for up to 4 extra books like FRT,GLS, etc.) if len(self) <= (39 + 4) and self.containsAnyOT39Books( ) and not self.containsAnyNT27Books(): testament, startBBB, endBBB = 'OT', 'GEN', 'MAL' booksExpected, textLineCountExpected, checkTotals = 39, 23145, theWordOTBookLines elif len(self) <= (27 + 4) and self.containsAnyNT27Books( ) and not self.containsAnyOT39Books(): testament, startBBB, endBBB = 'NT', 'MAT', 'REV' booksExpected, textLineCountExpected, checkTotals = 27, 7957, theWordNTBookLines else: # assume it's an entire Bible testament, startBBB, endBBB = 'BOTH', 'GEN', 'REV' booksExpected, textLineCountExpected, checkTotals = 66, 31102, theWordBookLines extension = '.bbl.mybible' vPrint('Info', debuggingThisModule, _(" Exporting to MySword format…")) mySettings = {} mySettings['unhandledMarkers'] = set() handledBooks = [] if 'MySwordOutputFilename' in controlDict: filename = controlDict['MySwordOutputFilename'] elif self.sourceFilename: filename = self.sourceFilename elif self.shortName: filename = self.shortName elif self.abbreviation: filename = self.abbreviation elif self.name: filename = self.name else: filename = 'export' if not filename.endswith(extension): filename += extension # Make sure that we have the right file extension filepath = os.path.join(outputFolder, BibleOrgSysGlobals.makeSafeFilename(filename)) if os.path.exists(filepath): os.remove(filepath) vPrint('Info', debuggingThisModule, ' createMySwordModule: ' + _("Writing {!r}…").format(filepath)) conn = sqlite3.connect(filepath) cursor = conn.cursor() # First write the settings Details table exeStr = 'CREATE TABLE Details(Description NVARCHAR(255), Abbreviation NVARCHAR(50), Comments TEXT, Version TEXT, VersionDate DATETIME, PublishDate DATETIME, RightToLeft BOOL, OT BOOL, NT BOOL, Strong BOOL' # incomplete customCSS = self.getSetting('CustomCSS') if customCSS: exeStr += ', CustomCSS TEXT' exeStr += ')' cursor.execute(exeStr) values = [] description = self.getSetting('Description') if not description: description = self.getSetting('description') if not description: description = self.name values.append(description) if self.abbreviation: abbreviation = self.abbreviation else: abbreviation = self.getSetting('WorkAbbreviation') if not abbreviation: abbreviation = self.name[:3].upper() values.append(abbreviation) comments = self.getSetting('Comments') values.append(comments) version = self.getSetting('Version') values.append(version) versionDate = self.getSetting('VersionDate') values.append(versionDate) publishDate = self.getSetting('PublishDate') values.append(publishDate) rightToLeft = self.getSetting('RightToLeft') values.append(rightToLeft) values.append(True if testament == 'OT' or testament == 'BOTH' else False) values.append(True if testament == 'NT' or testament == 'BOTH' else False) Strong = self.getSetting('Strong') values.append(Strong if Strong else False) if customCSS: values.append(customCSS) exeStr = 'INSERT INTO "Details" VALUES(' + '?,' * (len(values) - 1) + '?)' #dPrint( 'Quiet', debuggingThisModule, exeStr, values ) cursor.execute(exeStr, values) #if BibleOrgSysGlobals.debugFlag: cursor.execute( exeStr, values ) #else: # Not debugging #try: cursor.execute( exeStr, values ) #except sqlite3.InterfaceError: #logging.critical( "SQLite3 Interface error executing {} with {}".format( exeStr, values ) ) # Now create and fill the Bible table cursor.execute( 'CREATE TABLE Bible(Book INT, Chapter INT, Verse INT, Scripture TEXT, Primary Key(Book,Chapter,Verse))' ) conn.commit() # save (commit) the changes BBB, lineCount = startBBB, 0 while True: # Write each Bible book in the KJV order writeMSBook(cursor, BBB, mySettings) conn.commit() # save (commit) the changes handledBooks.append(BBB) if BBB == endBBB: break BBB = BOS.getNextBookCode(BBB) conn.commit() # save (commit) the changes cursor.close() if mySettings['unhandledMarkers']: logging.warning( "BibleWriter.createMySwordModule: Unhandled markers were {}". format(mySettings['unhandledMarkers'])) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled createMySwordModule markers were {}").format( mySettings['unhandledMarkers'])) unhandledBooks = [] for BBB in self.getBookList(): if BBB not in handledBooks: unhandledBooks.append(BBB) if unhandledBooks: logging.warning("createMySwordModule: Unhandled books were {}".format( unhandledBooks)) vPrint( 'Normal', debuggingThisModule, " " + _("WARNING: Unhandled createMySwordModule books were {}").format( unhandledBooks)) # Now create the gzipped file vPrint('Info', debuggingThisModule, " Compressing {} MySword file…".format(filename)) tar = tarfile.open(filepath + '.gz', 'w:gz') tar.add(filepath) tar.close() if BibleOrgSysGlobals.verbosityLevel > 0 and BibleOrgSysGlobals.maxProcesses > 1: vPrint('Quiet', debuggingThisModule, " BibleWriter.createMySwordModule finished successfully.") return True
class MySwordBible(Bible): """ Class for reading, validating, and converting MySwordBible files. """ def __init__(self, sourceFolder, givenFilename, encoding='utf-8') -> None: """ Constructor: just sets up the Bible object. """ # Setup and initialise the base class first Bible.__init__(self) self.objectNameString = 'MySword Bible object' self.objectTypeString = 'MySword' # Now we can set our object variables self.sourceFolder, self.sourceFilename, self.encoding = sourceFolder, givenFilename, encoding self.sourceFilepath = os.path.join(self.sourceFolder, self.sourceFilename) # Do a preliminary check on the readability of our file if not os.access(self.sourceFilepath, os.R_OK): logging.critical( _("MySwordBible: File {!r} is unreadable").format( self.sourceFilepath)) filenameBits = os.path.splitext(self.sourceFilename) self.name = filenameBits[0] self.fileExtension = filenameBits[1] #if self.fileExtension.upper().endswith('X'): #logging.warning( _("MySwordBible: File {!r} is encrypted").format( self.sourceFilepath ) ) # end of MySwordBible.__init__ def preload(self): """ Load the metadata from the SQLite3 database. """ fnPrint(debuggingThisModule, "preload()…") vPrint('Info', debuggingThisModule, _("Preloading {}…").format(self.sourceFilepath)) fileExtensionUpper = self.fileExtension.upper() if fileExtensionUpper not in FILENAME_ENDINGS_TO_ACCEPT: logging.critical("{} doesn't appear to be a MySword file".format( self.sourceFilename)) elif not self.sourceFilename.upper().endswith( BIBLE_FILENAME_ENDINGS_TO_ACCEPT[0]): logging.critical( "{} doesn't appear to be a MySword Bible file".format( self.sourceFilename)) connection = sqlite3.connect(self.sourceFilepath) connection.row_factory = sqlite3.Row # Enable row names self.cursor = connection.cursor() # First get the settings if self.suppliedMetadata is None: self.suppliedMetadata = {} self.suppliedMetadata['MySword'] = {} self.cursor.execute('select * from Details') row = self.cursor.fetchone() for key in row.keys(): self.suppliedMetadata['MySword'][key] = row[key] #dPrint( 'Quiet', debuggingThisModule, self.suppliedMetadata['MySword'] ); halt #if 'Description' in self.settingsDict and len(self.settingsDict['Description'])<40: self.name = self.settingsDict['Description'] #if 'Abbreviation' in self.settingsDict: self.abbreviation = self.settingsDict['Abbreviation'] if 'encryption' in self.suppliedMetadata['MySword']: logging.critical("{} is encrypted: level {}".format( self.sourceFilename, self.suppliedMetadata['MySword']['encryption'])) self.BibleOrganisationalSystem = BibleOrganisationalSystem( 'GENERIC-KJV-66-ENG') self.preloadDone = True # end of MySwordBible.preload def load(self): """ Load all the books out of the SQLite3 database. """ fnPrint(debuggingThisModule, "load()…") assert self.preloadDone vPrint('Info', debuggingThisModule, _("Loading {}…").format(self.sourceFilepath)) if self.suppliedMetadata['MySword']['OT'] and self.suppliedMetadata[ 'MySword']['NT']: testament, BBB = 'BOTH', 'GEN' booksExpected, textLineCountExpected = 66, 31102 elif self.suppliedMetadata['MySword']['OT']: testament, BBB = 'OT', 'GEN' booksExpected, textLineCountExpected = 39, 23145 elif self.suppliedMetadata['MySword']['NT']: testament, BBB = 'NT', 'MAT' booksExpected, textLineCountExpected = 27, 7957 # Create the first book thisBook = BibleBook(self, BBB) thisBook.objectNameString = 'MySword Bible Book object' thisBook.objectTypeString = 'MySword' verseList = self.BibleOrganisationalSystem.getNumVersesList(BBB) numC, numV = len(verseList), verseList[0] nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB) C = V = 1 bookCount = 0 ourGlobals = {} continued = ourGlobals['haveParagraph'] = False haveLines = False while True: self.cursor.execute( 'select Scripture from Bible where Book=? and Chapter=? and Verse=?', (nBBB, C, V)) try: row = self.cursor.fetchone() line = row[0] except TypeError: # This reference is missing (row is None) #dPrint( 'Quiet', debuggingThisModule, "something wrong at", BBB, C, V ) #if BibleOrgSysGlobals.debugFlag: halt #dPrint( 'Quiet', debuggingThisModule, row ) line = None #dPrint( 'Quiet', debuggingThisModule, nBBB, BBB, C, V, 'MySw file line is "' + line + '"' ) if line is None: logging.warning( "MySwordBible.load: Have missing verse line at {} {}:{}". format(BBB, C, V)) else: # line is not None if not isinstance(line, str): if 'encryption' in self.suppliedMetadata['MySword']: logging.critical( "MySwordBible.load: Unable to decrypt verse line at {} {}:{} {!r}" .format(BBB, C, V, line)) break else: logging.critical( "MySwordBible.load: Unable to decode verse line at {} {}:{} {!r} {}" .format(BBB, C, V, line, self.suppliedMetadata['MySword'])) elif not line: logging.warning( "MySwordBible.load: Found blank verse line at {} {}:{}" .format(BBB, C, V)) else: haveLines = True # Some modules end lines with \r\n or have it in the middle! # (We just ignore these for now) while line and line[-1] in '\r\n': line = line[:-1] if '\r' in line or '\n' in line: # (in the middle) logging.warning( "MySwordBible.load: Found CR or LF characters in verse line at {} {}:{}" .format(BBB, C, V)) line = line.replace('\r\n', ' ').replace('\r', ' ').replace('\n', ' ') #dPrint( 'Quiet', debuggingThisModule, "MySword.load", BBB, C, V, repr(line) ) handleRTFLine(self.name, BBB, C, V, line, thisBook, ourGlobals) V += 1 if V > numV: C += 1 if C > numC: # Save this book now if haveLines: vPrint('Verbose', debuggingThisModule, " MySword saving", BBB, bookCount + 1) self.stashBook(thisBook) #else: vPrint( 'Quiet', debuggingThisModule, "Not saving", BBB ) bookCount += 1 # Not the number saved but the number we attempted to process if bookCount >= booksExpected: break BBB = self.BibleOrganisationalSystem.getNextBookCode(BBB) # Create the next book thisBook = BibleBook(self, BBB) thisBook.objectNameString = 'MySword Bible Book object' thisBook.objectTypeString = 'MySword' haveLines = False verseList = self.BibleOrganisationalSystem.getNumVersesList( BBB) numC, numV = len(verseList), verseList[0] nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber( BBB) C = V = 1 #thisBook.addLine( 'c', str(C) ) else: # next chapter only #thisBook.addLine( 'c', str(C) ) numV = verseList[C - 1] V = 1 if ourGlobals['haveParagraph']: thisBook.addLine('p', '') ourGlobals['haveParagraph'] = False self.cursor.close() del self.cursor self.applySuppliedMetadata('MySword') # Copy some to self.settingsDict self.doPostLoadProcessing() # end of MySwordBible.load def loadBook(self, BBB: str): """ Load the requested book out of the SQLite3 database. """ fnPrint(debuggingThisModule, "loadBook( {} )".format(BBB)) assert self.preloadDone if BBB in self.books: dPrint('Quiet', debuggingThisModule, " {} is already loaded -- returning".format(BBB)) return # Already loaded if BBB in self.triedLoadingBook: logging.warning( "We had already tried loading MySwordBible {} for {}".format( BBB, self.name)) return # We've already attempted to load this book self.triedLoadingBook[BBB] = True self.bookNeedsReloading[BBB] = False if BibleOrgSysGlobals.verbosityLevel > 2 or BibleOrgSysGlobals.debugFlag: vPrint( 'Quiet', debuggingThisModule, _("MySwordBible: Loading {} from {}…").format( BBB, self.sourceFilepath)) #if self.suppliedMetadata['MySword']['OT'] and self.suppliedMetadata['MySword']['NT']: #testament, BBB = 'BOTH', 'GEN' #booksExpected, textLineCountExpected = 1, 31102 #elif self.suppliedMetadata['MySword']['OT']: #testament, BBB = 'OT', 'GEN' #booksExpected, textLineCountExpected = 1, 23145 #elif self.suppliedMetadata['MySword']['NT']: #testament, BBB = 'NT', 'MAT' #booksExpected, textLineCountExpected = 1, 7957 # Create the first book thisBook = BibleBook(self, BBB) thisBook.objectNameString = 'MySword Bible Book object' thisBook.objectTypeString = 'MySword' verseList = self.BibleOrganisationalSystem.getNumVersesList(BBB) numC, numV = len(verseList), verseList[0] nBBB = BibleOrgSysGlobals.loadedBibleBooksCodes.getReferenceNumber(BBB) C = V = 1 #bookCount = 0 ourGlobals = {} continued = ourGlobals['haveParagraph'] = False haveLines = False while True: self.cursor.execute( 'select Scripture from Bible where Book=? and Chapter=? and Verse=?', (nBBB, C, V)) try: row = self.cursor.fetchone() line = row[0] except TypeError: # This reference is missing (row is None) #dPrint( 'Quiet', debuggingThisModule, "something wrong at", BBB, C, V ) #if BibleOrgSysGlobals.debugFlag: halt #dPrint( 'Quiet', debuggingThisModule, row ) line = None #dPrint( 'Quiet', debuggingThisModule, nBBB, BBB, C, V, 'MySw file line is "' + line + '"' ) if line is None: logging.warning( "MySwordBible.load: Have missing verse line at {} {}:{}". format(BBB, C, V)) else: # line is not None if not isinstance(line, str): if 'encryption' in self.suppliedMetadata['MySword']: logging.critical( "MySwordBible.load: Unable to decrypt verse line at {} {}:{} {!r}" .format(BBB, C, V, line)) break else: logging.critical( "MySwordBible.load: Unable to decode verse line at {} {}:{} {!r} {}" .format(BBB, C, V, line, self.suppliedMetadata['MySword'])) elif not line: logging.warning( "MySwordBible.load: Found blank verse line at {} {}:{}" .format(BBB, C, V)) else: haveLines = True # Some modules end lines with \r\n or have it in the middle! # (We just ignore these for now) while line and line[-1] in '\r\n': line = line[:-1] if '\r' in line or '\n' in line: # (in the middle) logging.warning( "MySwordBible.load: Found CR or LF characters in verse line at {} {}:{}" .format(BBB, C, V)) line = line.replace('\r\n', ' ').replace('\r', ' ').replace('\n', ' ') #dPrint( 'Quiet', debuggingThisModule, "MySword.load", BBB, C, V, repr(line) ) handleRTFLine(self.name, BBB, C, V, line, thisBook, ourGlobals) V += 1 if V > numV: C += 1 if C <= numC: # next chapter only #thisBook.addLine( 'c', str(C) ) numV = verseList[C - 1] V = 1 else: # Save this book now if haveLines: vPrint('Info', debuggingThisModule, " MySword saving", BBB) self.stashBook(thisBook) #else: vPrint( 'Quiet', debuggingThisModule, "Not saving", BBB ) break if ourGlobals['haveParagraph']: thisBook.addLine('p', '') ourGlobals['haveParagraph'] = False