def DrupalBibleFileCheck(givenFolderName,
strictCheck: bool = True,
autoLoad: bool = False,
autoLoadBooks: bool = False):
"""
Given a folder, search for DrupalBible Bible files or folders in the folder and in the next level down.
Returns False if an error is found.
if autoLoad is false (default)
returns None, or the number of Bibles found.
if autoLoad is true and exactly one DrupalBible Bible is found,
returns the loaded DrupalBible object.
"""
fnPrint(
debuggingThisModule, "DrupalBibleFileCheck( {}, {}, {}, {} )".format(
givenFolderName, strictCheck, autoLoad, autoLoadBooks))
if BibleOrgSysGlobals.debugFlag:
assert givenFolderName and isinstance(givenFolderName, (str, Path))
if BibleOrgSysGlobals.debugFlag: assert autoLoad in (
True,
False,
)
# Check that the given folder is readable
if not os.access(givenFolderName, os.R_OK):
logging.critical(
_("DrupalBibleFileCheck: Given {!r} folder is unreadable").format(
givenFolderName))
return False
if not os.path.isdir(givenFolderName):
logging.critical(
_("DrupalBibleFileCheck: Given {!r} path is not a folder").format(
givenFolderName))
return False
# Find all the files and folders in this folder
vPrint(
'Verbose', debuggingThisModule,
" DrupalBibleFileCheck: Looking for files in given {}".format(
givenFolderName))
foundFolders, foundFiles = [], []
for something in os.listdir(givenFolderName):
somepath = os.path.join(givenFolderName, something)
if os.path.isdir(somepath):
if something in BibleOrgSysGlobals.COMMONLY_IGNORED_FOLDERS:
continue # don't visit these directories
foundFolders.append(something)
elif os.path.isfile(somepath):
somethingUpper = something.upper()
somethingUpperProper, somethingUpperExt = os.path.splitext(
somethingUpper)
if somethingUpperExt in filenameEndingsToAccept:
foundFiles.append(something)
# See if there's an DrupalBible project here in this given folder
numFound = 0
lastFilenameFound = None
for thisFilename in sorted(foundFiles):
if thisFilename.endswith('.bc'):
if strictCheck or BibleOrgSysGlobals.strictCheckingFlag:
firstLine = BibleOrgSysGlobals.peekIntoFile(
thisFilename, givenFolderName)
if firstLine is None:
continue # seems we couldn't decode the file
if (not firstLine.startswith('\ufeff*Bible')) and (
not firstLine.startswith("*Bible")):
vPrint(
'Verbose', debuggingThisModule,
"DrupalBible (unexpected) first line was {!r} in {}".
format(firstLine, thisFilename))
continue
lastFilenameFound = thisFilename
numFound += 1
if numFound:
vPrint('Info', debuggingThisModule, "DrupalBibleFileCheck got",
numFound, givenFolderName, lastFilenameFound)
if numFound == 1 and (autoLoad or autoLoadBooks):
uB = DrupalBible(givenFolderName, lastFilenameFound[:-3]
) # Remove the end of the actual filename ".bc"
if autoLoadBooks: uB.load() # Load and process the file
return uB
return numFound
# Look one level down
numFound = 0
foundProjects = []
for thisFolderName in sorted(foundFolders):
tryFolderName = os.path.join(givenFolderName, thisFolderName + '/')
if not os.access(tryFolderName,
os.R_OK): # The subfolder is not readable
logging.warning(
_("DrupalBibleFileCheck: {!r} subfolder is unreadable").format(
tryFolderName))
continue
vPrint(
'Verbose', debuggingThisModule,
" DrupalBibleFileCheck: Looking for files in {}".format(
tryFolderName))
foundSubfolders, foundSubfiles = [], []
try:
for something in os.listdir(tryFolderName):
somepath = os.path.join(givenFolderName, thisFolderName,
something)
if os.path.isdir(somepath): foundSubfolders.append(something)
elif os.path.isfile(somepath):
somethingUpper = something.upper()
somethingUpperProper, somethingUpperExt = os.path.splitext(
somethingUpper)
if somethingUpperExt in filenameEndingsToAccept:
foundSubfiles.append(something)
except PermissionError:
pass # can't read folder, e.g., system folder
# See if there's an DrupalBible project here in this folder
for thisFilename in sorted(foundSubfiles):
if thisFilename.endswith('.bc'):
if strictCheck or BibleOrgSysGlobals.strictCheckingFlag:
firstLine = BibleOrgSysGlobals.peekIntoFile(
thisFilename, tryFolderName)
if firstLine is None:
continue # seems we couldn't decode the file
if (not firstLine.startswith('\ufeff*Bible')) and (
not firstLine.startswith("*Bible")):
vPrint(
'Verbose', debuggingThisModule,
"DrupalBible (unexpected) first line was {!r} in {}"
.format(firstLine, thisFilename))
halt
continue
#dPrint( 'Quiet', debuggingThisModule, "BFC_here", repr(tryFolderName), repr(thisFilename) )
foundProjects.append((
tryFolderName,
thisFilename,
))
lastFilenameFound = thisFilename
numFound += 1
if numFound:
vPrint('Info', debuggingThisModule,
"DrupalBibleFileCheck foundProjects", numFound, foundProjects)
if numFound == 1 and (autoLoad or autoLoadBooks):
if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1
uB = DrupalBible(
foundProjects[0][0], foundProjects[0][1]
[:-3]) # Remove the end of the actual filename ".bc"
if autoLoadBooks: uB.load() # Load and process the file
return uB
return numFound
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
if 1: # Try the default one
vPrint('Quiet', debuggingThisModule,
"\nTrying default Bible stylesheet…")
#folder = Path( '/mnt/SSDs/Work/VirtualBox_Shared_Folder/PTStylesheets/' )
#filename = "LD.sty"
ss = BibleStylesheet()
#vPrint( 'Quiet', debuggingThisModule, ss )
#ss.importParatextStylesheet( folder, filename, encoding='latin-1' )
ss.loadDefault()
vPrint('Quiet', debuggingThisModule, ss)
vPrint('Quiet', debuggingThisModule, "h style:",
ss.getTKStyleDict('h'))
try:
vPrint('Quiet', debuggingThisModule, "s1 font:",
ss.getValue('s1', 'font'))
except KeyError:
vPrint('Quiet', debuggingThisModule,
"No s1 or font in stylesheet!")
try:
vPrint('Quiet', debuggingThisModule, ss.getTKStyleDict('hijkl'))
except KeyError:
vPrint('Quiet', debuggingThisModule, "No hijkl in stylesheet!")
try:
vPrint('Quiet', debuggingThisModule,
ss.getValue('h', 'FontSizeImaginary'))
except KeyError:
vPrint('Quiet', debuggingThisModule,
"No h or FontSizeImaginary in stylesheet!")
vPrint('Never', debuggingThisModule, ss.dataDict)
if 1: # Try importing one
vPrint('Quiet', debuggingThisModule,
"\nTrying Bible stylesheet import…")
folder = Path('/mnt/SSDs/Work/VirtualBox_Shared_Folder/PTStylesheets/')
filename = "LD.sty"
ss = BibleStylesheet()
vPrint('Quiet', debuggingThisModule, ss)
ss.importParatextStylesheet(folder, filename, encoding='latin-1')
vPrint('Quiet', debuggingThisModule, ss)
vPrint('Never', debuggingThisModule, ss.dataDict)
vPrint('Quiet', debuggingThisModule, "h style:",
ss.getTKStyleDict('h'))
try:
vPrint('Quiet', debuggingThisModule, "h FontSize:",
ss.getValue('h', 'FontSize'))
except KeyError:
vPrint('Quiet', debuggingThisModule,
"No h or FontSize in stylesheet!")
if 1: # Try a small one
vPrint('Quiet', debuggingThisModule, "\nTrying small PT stylesheet…")
folder = Path('/mnt/SSDs/Work/VirtualBox_Shared_Folder/PTStylesheets/')
filename = "LD.sty"
ss = ParatextStylesheet().load(folder, filename, encoding='latin-1')
vPrint('Quiet', debuggingThisModule, ss)
vPrint('Quiet', debuggingThisModule, "h style:", ss.getDict('h'))
vPrint('Quiet', debuggingThisModule, "h fontsize:",
ss.getValue('h', 'FontSize'))
vPrint('Never', debuggingThisModule, ss.dataDict)
if 1: # Try a full one
vPrint('Quiet', debuggingThisModule, "\nTrying full PT stylesheet…")
folder = Path('/mnt/SSDs/Work/VirtualBox_Shared_Folder/PTStylesheets/')
filename = "usfm.sty"
ss = ParatextStylesheet()
ss.load(folder, filename)
vPrint('Quiet', debuggingThisModule, ss)
vPrint('Quiet', debuggingThisModule, "h style:", ss.getDict('h'))
vPrint('Quiet', debuggingThisModule, "h fontsize:",
ss.getValue('h', 'FontSize'))
vPrint('Never', debuggingThisModule, ss.dataDict)
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
testFolder = BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath(
'DrupalTest/')
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
result1 = DrupalBibleFileCheck(testFolder)
vPrint('Normal', debuggingThisModule, "DrupalBible TestA1", result1)
result2 = DrupalBibleFileCheck(testFolder, autoLoad=True)
vPrint('Normal', debuggingThisModule, "DrupalBible TestA2", result2)
result3 = DrupalBibleFileCheck(testFolder, autoLoadBooks=True)
vPrint('Normal', debuggingThisModule, "DrupalBible TestA3", result3)
#testSubfolder = os.path.join( testFolder, 'kjv/' )
#result3 = DrupalBibleFileCheck( testSubfolder )
#dPrint( 'Normal', debuggingThisModule, "DrupalBible TestB1", result3 )
#result4 = DrupalBibleFileCheck( testSubfolder, autoLoad=True )
#dPrint( 'Normal', debuggingThisModule, "DrupalBible TestB2", result4 )
if 1: # specified modules
single = ('kjv', )
good = ('kjv', )
nonEnglish = ()
bad = ()
for j, testFilename in enumerate(
good
): # Choose one of the above: single, good, nonEnglish, bad
vPrint('Normal', debuggingThisModule,
"\nDrupalBible C{}/ Trying {}".format(j + 1, testFilename))
#myTestFolder = os.path.join( testFolder, testFilename+'/' )
#testFilepath = os.path.join( testFolder, testFilename+'/', testFilename+'_utf8.txt' )
testDB(testFilename)
if 1: # all discovered modules in the test folder
foundFolders, foundFiles = [], []
for something in os.listdir(testFolder):
somepath = os.path.join(testFolder, something)
if os.path.isdir(somepath): foundFolders.append(something)
elif os.path.isfile(somepath): foundFiles.append(something)
if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work
vPrint(
'Normal', debuggingThisModule,
"\nTrying all {} discovered modules…".format(
len(foundFolders)))
parameters = [folderName for folderName in sorted(foundFolders)]
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses
) as pool: # start worker processes
results = pool.map(testDB,
parameters) # have the pool do our loads
assert len(results) == len(
parameters
) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate(sorted(foundFolders)):
vPrint(
'Normal', debuggingThisModule,
"\nDrupalBible D{}/ Trying {}".format(j + 1, someFolder))
#myTestFolder = os.path.join( testFolder, someFolder+'/' )
testDB(someFolder)
def loadSystems(self, XMLFolder=None):
"""
Load and pre-process the specified punctuation systems.
"""
if not self._XMLSystems: # Only ever do this once
if XMLFolder is None:
XMLFolder = BibleOrgSysGlobals.BOS_DATAFILES_FOLDERPATH.joinpath(
'PunctuationSystems/') # Relative to module, not cwd
self.__XMLFolder = XMLFolder
vPrint(
'Info', debuggingThisModule,
_("Loading punctuations systems from {}…").format(
self.__XMLFolder))
filenamePrefix = "BIBLEPUNCTUATIONSYSTEM_"
for filename in os.listdir(self.__XMLFolder):
filepart, extension = os.path.splitext(filename)
if extension.upper() == '.XML' and filepart.upper().startswith(
filenamePrefix):
punctuationSystemCode = filepart[len(filenamePrefix):]
vPrint(
'Verbose', debuggingThisModule,
_("Loading {} punctuation system from {}…").format(
punctuationSystemCode, filename))
self._XMLSystems[punctuationSystemCode] = {}
self._XMLSystems[punctuationSystemCode][
'tree'] = ElementTree().parse(
os.path.join(self.__XMLFolder, filename))
assert self._XMLSystems[punctuationSystemCode][
'tree'] # Fail here if we didn't load anything at all
# Check and remove the header element
if self._XMLSystems[punctuationSystemCode][
'tree'].tag == self.XMLTreeTag:
header = self._XMLSystems[punctuationSystemCode][
'tree'][0]
if header.tag == self.headerTag:
self._XMLSystems[punctuationSystemCode][
'header'] = header
self._XMLSystems[punctuationSystemCode][
'tree'].remove(header)
BibleOrgSysGlobals.checkXMLNoText(header, 'header')
BibleOrgSysGlobals.checkXMLNoTail(header, 'header')
BibleOrgSysGlobals.checkXMLNoAttributes(
header, 'header')
if len(header) > 1:
logging.info(
_("Unexpected elements in header"))
elif len(header) == 0:
logging.info(
_("Missing work element in header"))
else:
work = header[0]
BibleOrgSysGlobals.checkXMLNoText(
work, "work in header")
BibleOrgSysGlobals.checkXMLNoTail(
work, "work in header")
BibleOrgSysGlobals.checkXMLNoAttributes(
work, "work in header")
if work.tag == "work":
self._XMLSystems[punctuationSystemCode][
'version'] = work.find('version').text
self._XMLSystems[punctuationSystemCode][
'date'] = work.find('date').text
self._XMLSystems[punctuationSystemCode][
'title'] = work.find('title').text
else:
logging.warning(
_("Missing work element in header"))
else:
logging.warning(
_("Missing header element (looking for {!r} tag)"
).format(self.headerTag))
else:
logging.error(
_("Expected to load {!r} but got {!r}").format(
self.XMLTreeTag,
self._XMLSystems[punctuationSystemCode]
['tree'].tag))
bookCount = 0 # There must be an easier way to do this
for subelement in self._XMLSystems[punctuationSystemCode][
'tree']:
bookCount += 1
vPrint(
'Info', debuggingThisModule,
_(" Loaded {} books for {}").format(
bookCount, punctuationSystemCode))
logging.info(
_(" Loaded {} books for {}").format(
bookCount, punctuationSystemCode))
if BibleOrgSysGlobals.strictCheckingFlag:
self._validateSystem(
self._XMLSystems[punctuationSystemCode]['tree'],
punctuationSystemCode)
return self
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
sampleBookList = [
'GEN', 'JDG', 'SA1', 'SA2', 'KI1', 'KI2', 'MA4', 'MAT', 'MRK', 'LUK',
'JHN', 'ACT', 'ROM', 'CO1', 'CO2', 'PE1', 'PE2', 'JDE', 'REV'
]
#sampleBookList = ['GEN','JDG','SA1','SA2','KI1','KI2','MA1','MA2']
#sampleBookList = ['MAT','MRK','LUK','JHN','ACT','ROM','CO1','CO2','GAL','EPH','PHP','COL','PE1','PE2','JDE','REV']
# Demo the BibleBooksNamesSystems object
bbnss = BibleBooksNamesSystems().loadData(
) # Doesn't reload the XML unnecessarily :)
vPrint('Quiet', debuggingThisModule, bbnss) # Just print a summary
vPrint('Quiet', debuggingThisModule, "Available system names are:",
bbnss.getAvailableBooksNamesSystemNames())
vPrint('Quiet', debuggingThisModule, "Available eng system names are:",
bbnss.getAvailableBooksNamesSystemNames(
'eng')) # Just get the ones for this language code
vPrint('Quiet', debuggingThisModule, "Available mbt system names are:",
bbnss.getAvailableBooksNamesSystemNames(languageCode='mbt'))
vPrint('Quiet', debuggingThisModule, "Available language codes are:",
bbnss.getAvailableLanguageCodes())
for bookName in ('Genesis', 'Genèse', 'Génesis', 'Gênesis', '1 John'):
vPrint(
'Quiet', debuggingThisModule,
"From {!r} got {}".format(bookName,
bbnss.getBBBFromText(bookName)))
# Demo the BibleBooksNamesSystem object
bbns1 = BibleBooksNamesSystem(
"eng_traditional") # Doesn't reload the XML unnecessarily :)
vPrint('Quiet', debuggingThisModule, bbns1) # Just print a summary
# Demo the BibleBooksNamesSystem object with a book list
bbns2 = BibleBooksNamesSystem(
"eng_traditional",
sampleBookList) # Doesn't reload the XML unnecessarily :)
vPrint('Quiet', debuggingThisModule, bbns2) # Just print a summary
vPrint('Quiet', debuggingThisModule, "Checking book name inputs…")
for bookAbbrevInput in ('Gen', 'GEN', 'Gn', 'Exo', 'Judges', '1 Samuel',
'1Samuel', '1Sam', '1 Sam', '1 Sml', '1Sml', '1 S',
'1S', 'II Sa', 'IIS', '1Kgs', '1 Kgs', '1K', '1 K',
'IK', 'I K', '1M', 'IV Mac', 'Mt', 'Jude', 'Rvl'):
# NOTE: '1S' is ambiguous with '1st' :(
vPrint(
'Quiet', debuggingThisModule, " Searching for {!r} got {}".format(
bookAbbrevInput, bbns2.getBBBFromText(bookAbbrevInput)))
vPrint('Quiet', debuggingThisModule, "Checking division name inputs…")
for divisionAbbrevInput in ('OT', 'NewTest', 'Paul', 'Deutero', 'Gn',
'Exo'): # Last two should always fail
vPrint(
'Quiet', debuggingThisModule, " Searching for {!r} got {}".format(
divisionAbbrevInput,
bbns2.getDivisionAbbreviation(divisionAbbrevInput)))
vPrint('Quiet', debuggingThisModule, "Getting division booklists…")
for divisionAbbrevInput in ('OT', 'NT', 'NewTest', 'Paul', 'Deutero', 'Gn',
'Exo', '1 Samuel'):
vPrint(
'Quiet', debuggingThisModule, " Searching for {!r} got {}".format(
divisionAbbrevInput,
bbns2.getDivisionBooklist(divisionAbbrevInput)))
def __validate(self):
"""
Check/validate the loaded data.
"""
assert self._XMLTree
uniqueDict = {}
for elementName in self._uniqueElements:
uniqueDict["Element_" + elementName] = []
for attributeName in self._uniqueAttributes:
uniqueDict["Attribute_" + attributeName] = []
expectedID = 1
for j, element in enumerate(self._XMLTree):
if element.tag == self._mainElementTag:
BibleOrgSysGlobals.checkXMLNoText(element, element.tag)
BibleOrgSysGlobals.checkXMLNoTail(element, element.tag)
if not self._compulsoryAttributes and not self._optionalAttributes:
BibleOrgSysGlobals.checkXMLNoAttributes(
element, element.tag)
if not self._compulsoryElements and not self._optionalElements:
BibleOrgSysGlobals.checkXMLNoSubelements(
element, element.tag)
# Check compulsory attributes on this main element
for attributeName in self._compulsoryAttributes:
attributeValue = element.get(attributeName)
if attributeValue is None:
logging.error(
_("Compulsory {!r} attribute is missing from {} element in record {}"
).format(attributeName, element.tag, j))
if not attributeValue:
logging.warning(
_("Compulsory {!r} attribute is blank on {} element in record {}"
).format(attributeName, element.tag, j))
# Check optional attributes on this main element
for attributeName in self._optionalAttributes:
attributeValue = element.get(attributeName)
if attributeValue is not None:
if not attributeValue:
logging.warning(
_("Optional {!r} attribute is blank on {} element in record {}"
).format(attributeName, element.tag, j))
# Check for unexpected additional attributes on this main element
for attributeName in element.keys():
attributeValue = element.get(attributeName)
if attributeName not in self._compulsoryAttributes and attributeName not in self._optionalAttributes:
logging.warning(
_("Additional {!r} attribute ({!r}) found on {} element in record {}"
).format(attributeName, attributeValue,
element.tag, j))
# Check the attributes that must contain unique information (in that particular field -- doesn't check across different attributes)
for attributeName in self._uniqueAttributes:
attributeValue = element.get(attributeName)
if attributeValue is not None:
if attributeValue in uniqueDict["Attribute_" +
attributeName]:
logging.error(
_("Found {!r} data repeated in {!r} field on {} element in record {}"
).format(attributeValue, attributeName,
element.tag, j))
uniqueDict["Attribute_" +
attributeName].append(attributeValue)
# Get the marker to use as a record ID
marker = element.find("marker").text
# Check compulsory elements
for elementName in self._compulsoryElements:
if element.find(elementName) is None:
logging.error(
_("Compulsory {!r} element is missing in record with marker {!r} (record {})"
).format(elementName, marker, j))
elif not element.find(elementName).text:
logging.warning(
_("Compulsory {!r} element is blank in record with marker {!r} (record {})"
).format(elementName, marker, j))
# Check optional elements
for elementName in self._optionalElements:
if element.find(elementName) is not None:
if not element.find(elementName).text:
logging.warning(
_("Optional {!r} element is blank in record with marker {!r} (record {})"
).format(elementName, marker, j))
# Check for unexpected additional elements
for subelement in element:
if subelement.tag not in self._compulsoryElements and subelement.tag not in self._optionalElements:
logging.warning(
_("Additional {!r} element ({!r}) found in record with marker {!r} (record {})"
).format(subelement.tag, subelement.text, marker,
j))
# Check the elements that must contain unique information (in that particular element -- doesn't check across different elements)
for elementName in self._uniqueElements:
if element.find(elementName) is not None:
text = element.find(elementName).text
if text in uniqueDict["Element_" + elementName]:
logging.error(
_("Found {!r} data repeated in {!r} element in record with marker {!r} (record {})"
).format(text, elementName, marker, j))
uniqueDict["Element_" + elementName].append(text)
else:
logging.warning(
_("Unexpected element: {} in record {}").format(
element.tag, j))
if element.tail is not None and element.tail.strip():
logging.error(
_("Unexpected {!r} tail data after {} element in record {}"
).format(element.tail, element.tag, j))
if self._XMLTree.tail is not None and self._XMLTree.tail.strip():
logging.error(
_("Unexpected {!r} tail data after {} element").format(
self._XMLTree.tail, self._XMLTree.tag))
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
# Demo the BibleBooksCodes object
bbc = BibleBooksCodes().loadData(
) # Doesn't reload the XML unnecessarily :)
vPrint('Quiet', debuggingThisModule, bbc) # Just print a summary
vPrint(
'Quiet', debuggingThisModule, "Esther has {} expected chapters".format(
bbc.getExpectedChaptersList("EST")))
vPrint(
'Quiet', debuggingThisModule,
"Apocalypse of Ezra has {} expected chapters".format(
bbc.getExpectedChaptersList("EZA")))
vPrint('Quiet', debuggingThisModule,
"Psalms has {} expected chapters".format(bbc.getMaxChapters("PSA")))
vPrint('Quiet', debuggingThisModule, "Names for Genesis are:",
bbc.getEnglishNameList_NR("GEN"))
vPrint('Quiet', debuggingThisModule, "Names for Sirach are:",
bbc.getEnglishNameList_NR('SIR'))
vPrint('Quiet', debuggingThisModule, "All BBBs:",
len(bbc.getAllReferenceAbbreviations()),
bbc.getAllReferenceAbbreviations())
vPrint('Quiet', debuggingThisModule, "All BBBs in a print sequence",
len(bbc.getSequenceList()), bbc.getSequenceList())
myBBBs = ['GEN', 'EXO', 'PSA', 'ISA', 'MAL', 'MAT', 'REV', 'GLS']
vPrint('Quiet', debuggingThisModule, "My BBBs in sequence", len(myBBBs),
myBBBs, "now", len(bbc.getSequenceList(myBBBs)),
bbc.getSequenceList(myBBBs))
for BBB in myBBBs:
vPrint(
'Quiet', debuggingThisModule,
"{} is typically in {} section".format(BBB,
bbc.getTypicalSection(BBB)))
myBBBs = [
'REV', 'CO2', 'GEN', 'PSA', 'CO1', 'ISA', 'SA2', 'MAT', 'GLS', 'JOB'
]
vPrint('Quiet', debuggingThisModule, "My BBBs in sequence", len(myBBBs),
myBBBs, "now", len(bbc.getSequenceList(myBBBs)),
bbc.getSequenceList(myBBBs))
for BBB in myBBBs:
vPrint(
'Quiet', debuggingThisModule,
"{} is typically in {} section".format(BBB,
bbc.getTypicalSection(BBB)))
vPrint('Quiet', debuggingThisModule, "USFM triples:",
len(bbc.getAllUSFMBooksCodeNumberTriples()),
bbc.getAllUSFMBooksCodeNumberTriples())
vPrint('Quiet', debuggingThisModule, "USX triples:",
len(bbc.getAllUSXBooksCodeNumberTriples()),
bbc.getAllUSXBooksCodeNumberTriples())
vPrint('Quiet', debuggingThisModule, "Bibledit triples:",
len(bbc.getAllBibleditBooksCodeNumberTriples()),
bbc.getAllBibleditBooksCodeNumberTriples())
vPrint(
'Quiet', debuggingThisModule,
"Single chapter books (and OSIS):\n {}\n {}".format(
bbc.getSingleChapterBooksList(),
bbc.getOSISSingleChapterBooksList()))
vPrint(
'Quiet', debuggingThisModule,
"Possible alternative books to Esther: {}".format(
bbc.getPossibleAlternativeBooksCodes('EST')))
for something in (
'PE2',
'2Pe',
'2 Pet',
'2Pet',
'Job',
):
vPrint('Quiet', debuggingThisModule,
'{!r} -> {}'.format(something, bbc.getBBBFromText(something)))
myOSIS = (
'Gen',
'1Kgs',
'Ps',
'Mal',
'Matt',
'2John',
'Rev',
'EpLao',
'3Meq',
)
for osisCode in myOSIS:
vPrint(
'Quiet', debuggingThisModule,
"Osis {!r} -> {}".format(osisCode,
bbc.getBBBFromOSISAbbreviation(osisCode)))
sections: Dict[str, List[str]] = {}
for BBB in bbc:
section = bbc.getTypicalSection(BBB)
if section not in sections: sections[section] = []
sections[section].append(BBB)
vPrint('Quiet', debuggingThisModule,
"\n{} book codes in {} sections".format(len(bbc), len(sections)))
for section in sections:
vPrint(
'Quiet', debuggingThisModule,
" {} section: {} {}".format(section, len(sections[section]),
sections[section]))
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
testFolders = (
BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('CSVTest1/'),
BibleOrgSysGlobals.BOS_TEST_DATA_FOLDERPATH.joinpath('CSVTest2/'))
if 1: # demo the file checking code -- first with the whole folder and then with only one folder
for testFolder in testFolders:
result1 = CSVBibleFileCheck(testFolder)
vPrint('Normal', debuggingThisModule, "CSV TestA1", result1)
result2 = CSVBibleFileCheck(testFolder, autoLoad=True)
vPrint('Normal', debuggingThisModule, "CSV TestA2", result2)
result3 = CSVBibleFileCheck(testFolder, autoLoadBooks=True)
vPrint('Normal', debuggingThisModule, "CSV TestA3", result3)
#result3.loadMetadataFile( os.path.join( testFolder, "BooknamesMetadata.txt" ) )
if BibleOrgSysGlobals.strictCheckingFlag:
result3.check()
#dPrint( 'Quiet', debuggingThisModule, UsfmB.books['GEN']._processedLines[0:40] )
vBErrors = result3.getCheckResults()
#dPrint( 'Quiet', debuggingThisModule, vBErrors )
if BibleOrgSysGlobals.commandLineArguments.export:
##result3.toDrupalBible()
result3.doAllExports(wantPhotoBible=False,
wantODFs=False,
wantPDFs=False)
if 0: # all discovered modules in the test folder
foundFolders, foundFiles = [], []
for something in os.listdir(testFolder):
somepath = os.path.join(testFolder, something)
if os.path.isdir(somepath): foundFolders.append(something)
elif os.path.isfile(somepath): foundFiles.append(something)
if BibleOrgSysGlobals.maxProcesses > 1: # Get our subprocesses ready and waiting for work
vPrint(
'Normal', debuggingThisModule,
"\nTrying all {} discovered modules…".format(
len(foundFolders)))
parameters = [folderName for folderName in sorted(foundFolders)]
BibleOrgSysGlobals.alreadyMultiprocessing = True
with multiprocessing.Pool(processes=BibleOrgSysGlobals.maxProcesses
) as pool: # start worker processes
results = pool.map(testCSV,
parameters) # have the pool do our loads
assert len(results) == len(
parameters
) # Results (all None) are actually irrelevant to us here
BibleOrgSysGlobals.alreadyMultiprocessing = False
else: # Just single threaded
for j, someFolder in enumerate(sorted(foundFolders)):
vPrint('Normal', debuggingThisModule,
"\nCSV D{}/ Trying {}".format(j + 1, someFolder))
#myTestFolder = os.path.join( testFolder, someFolder+'/' )
testCSV(someFolder)
def CSVBibleFileCheck(givenFolderName,
strictCheck: bool = True,
autoLoad: bool = False,
autoLoadBooks: bool = False):
"""
Given a folder, search for CSV Bible files or folders in the folder and in the next level down.
Returns False if an error is found.
if autoLoad is false (default)
returns None, or the number of Bibles found.
if autoLoad is true and exactly one CSV Bible is found,
returns the loaded CSVBible object.
"""
fnPrint(
debuggingThisModule,
f"CSVBibleFileCheck( {givenFolderName}, {strictCheck}, {autoLoad}, {autoLoadBooks} )"
)
if BibleOrgSysGlobals.debugFlag:
assert givenFolderName and isinstance(givenFolderName, (str, Path))
assert autoLoad in (
True,
False,
) and autoLoadBooks in (
True,
False,
)
# Check that the given folder is readable
if not os.access(givenFolderName, os.R_OK):
logging.critical(
_("CSVBibleFileCheck: Given {!r} folder is unreadable").format(
givenFolderName))
return False
if not os.path.isdir(givenFolderName):
logging.critical(
_("CSVBibleFileCheck: Given {!r} path is not a folder").format(
givenFolderName))
return False
# Find all the files and folders in this folder
vPrint(
'Verbose', debuggingThisModule,
" CSVBibleFileCheck: Looking for files in given {!r}".format(
givenFolderName))
foundFolders, foundFiles = [], []
for something in os.listdir(givenFolderName):
somepath = os.path.join(givenFolderName, something)
if os.path.isdir(somepath):
if something in BibleOrgSysGlobals.COMMONLY_IGNORED_FOLDERS:
continue # don't visit these directories
foundFolders.append(something)
elif os.path.isfile(somepath):
somethingUpper = something.upper()
somethingUpperProper, somethingUpperExt = os.path.splitext(
somethingUpper)
ignore = False
for ending in filenameEndingsToIgnore:
if somethingUpper.endswith(ending):
ignore = True
break
if ignore: continue
if not somethingUpperExt[
1:] in extensionsToIgnore: # Compare without the first dot
foundFiles.append(something)
# See if there's an CSV Bible here in this given folder
numFound = 0
looksHopeful = False
lastFilenameFound = None
for thisFilename in sorted(foundFiles):
if thisFilename in ('book_names.txt', 'Readme.txt'):
looksHopeful = True
elif thisFilename.endswith('.txt'):
if strictCheck or BibleOrgSysGlobals.strictCheckingFlag:
firstLine = BibleOrgSysGlobals.peekIntoFile(
thisFilename, givenFolderName)
if firstLine is None:
continue # seems we couldn't decode the file
if not firstLine.startswith( '"Book","Chapter","Verse",' ) and not firstLine.startswith( '"1","1","1",') \
and not firstLine.startswith( 'Book,Chapter,Verse,' ) and not firstLine.startswith( '1,1,1,'):
vPrint(
'Verbose', debuggingThisModule,
"CSVBibleFileCheck: (unexpected) first line was {!r} in {}"
.format(firstLine, thisFilename))
continue
lastFilenameFound = thisFilename
numFound += 1
if numFound:
vPrint('Info', debuggingThisModule, "CSVBibleFileCheck got", numFound,
givenFolderName, lastFilenameFound)
if numFound == 1 and (autoLoad or autoLoadBooks):
uB = CSVBible(givenFolderName, lastFilenameFound[:-4]
) # Remove the end of the actual filename ".txt"
if autoLoadBooks: uB.load() # Load and process the file
return uB
return numFound
elif looksHopeful and BibleOrgSysGlobals.verbosityLevel > 2:
vPrint('Quiet', debuggingThisModule,
" Looked hopeful but no actual files found")
# Look one level down
numFound = 0
foundProjects = []
for thisFolderName in sorted(foundFolders):
tryFolderName = os.path.join(givenFolderName, thisFolderName + '/')
if not os.access(tryFolderName,
os.R_OK): # The subfolder is not readable
logging.warning(
_("CSVBibleFileCheck: {!r} subfolder is unreadable").format(
tryFolderName))
continue
vPrint(
'Verbose', debuggingThisModule,
" CSVBibleFileCheck: Looking for files in {}".format(
tryFolderName))
foundSubfolders, foundSubfiles = [], []
try:
for something in os.listdir(tryFolderName):
somepath = os.path.join(givenFolderName, thisFolderName,
something)
if os.path.isdir(somepath): foundSubfolders.append(something)
elif os.path.isfile(somepath):
somethingUpper = something.upper()
somethingUpperProper, somethingUpperExt = os.path.splitext(
somethingUpper)
ignore = False
for ending in filenameEndingsToIgnore:
if somethingUpper.endswith(ending):
ignore = True
break
if ignore: continue
if not somethingUpperExt[
1:] in extensionsToIgnore: # Compare without the first dot
foundSubfiles.append(something)
except PermissionError:
pass # can't read folder, e.g., system folder
# See if there's an CSV Bible here in this folder
for thisFilename in sorted(foundSubfiles):
if thisFilename.endswith('.txt'):
if strictCheck or BibleOrgSysGlobals.strictCheckingFlag:
firstLine = BibleOrgSysGlobals.peekIntoFile(
thisFilename, tryFolderName)
if firstLine is None:
continue # seems we couldn't decode the file
if not firstLine.startswith("Ge 1:1 "):
vPrint(
'Verbose', debuggingThisModule,
"CSVBibleFileCheck: (unexpected) first line was {!r} in {}"
.format(firstLine, thisFilename))
if debuggingThisModule: halt
continue
foundProjects.append((
tryFolderName,
thisFilename,
))
lastFilenameFound = thisFilename
numFound += 1
if numFound:
vPrint('Info', debuggingThisModule, "CSVBibleFileCheck foundProjects",
numFound, foundProjects)
if numFound == 1 and (autoLoad or autoLoadBooks):
if BibleOrgSysGlobals.debugFlag: assert len(foundProjects) == 1
uB = CSVBible(
foundProjects[0][0], foundProjects[0][1]
[:-4]) # Remove the end of the actual filename ".txt"
if autoLoadBooks: uB.load() # Load and process the file
return uB
return numFound
self.assertFalse(BAR.matchesAnchorString(ourAnchor))
# end of test_410_BibleAnchorReference
# end of BibleReferencesTests class
def fullDemo() -> None:
"""
Full demo to check class is working
"""
briefDemo()
# end of fullDemo
if __name__ == '__main__':
from multiprocessing import freeze_support
freeze_support() # Multiprocessing support for frozen Windows executables
# Configure basic set-up
parser = BibleOrgSysGlobals.setup(SHORT_PROGRAM_NAME, PROGRAM_VERSION,
LAST_MODIFIED_DATE)
BibleOrgSysGlobals.addStandardOptionsAndProcess(parser)
vPrint('Normal', debuggingThisModule, programNameVersion)
unittest.main() # Automatically runs all of the above tests
# end of BibleReferencesTests.py
def main() -> None:
"""
This is the main program for the app
which just tries to open and load some kind of Bible file(s)
from the inputFolder that you specified
and then export a PhotoBible (in the default BOSOutputFiles folder).
Note that the standard verbosityLevel is 2:
-s (silent) is 0
-q (quiet) is 1
-i (information) is 3
-v (verbose) is 4.
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
ourBibleOrganisationalSystem = BibleOrganisationalSystem(
"GENERIC-KJV-66-ENG")
ourVersificationSystem = ourBibleOrganisationalSystem.getVersificationSystemName(
)
ourBibleSingleReference = BibleSingleReference(
ourBibleOrganisationalSystem)
vPrint('Quiet', debuggingThisModule, _("Use QUIT or EXIT to finish."))
while True: # Loop until they stop it
userInput = input('\n' + _(
"Enter a verse number 1..31102 or a single Bible verse reference (or QUIT): "
))
if userInput.lower() in (
'exit',
'quit',
'q',
'stop',
'halt',
):
break
# See if it's an absolute verse number
try:
userInt = int(userInput)
except ValueError:
userInt = None
if userInt:
if 1 <= userInt <= 31102:
BBB, C, V = ourBibleOrganisationalSystem.convertAbsoluteVerseNumber(
userInt)
vPrint(
'Quiet', debuggingThisModule,
_("{} verse number {} is {} {}:{}").format(
ourVersificationSystem, userInt, BBB, C, V))
else:
vPrint('Quiet', debuggingThisModule,
_("Absolute verse numbers must be in range 1..31,102."))
else: # assume it's a Bible reference
adjustedUserInput = userInput
if ':' not in adjustedUserInput:
for alternative in (
'.',
',',
'-',
): # Handle possible alternative C:V punctuations
if alternative in adjustedUserInput:
adjustedUserInput = adjustedUserInput.replace(
alternative, ':', 1)
break
results = ourBibleSingleReference.parseReferenceString(
adjustedUserInput)
#vPrint( 'Quiet', debuggingThisModule, results )
successFlag, haveWarnings, BBB, C, V, S = results
if successFlag:
vPrint(
'Quiet', debuggingThisModule,
_("{!r} converted to {} {}:{} in our internal system.").
format(userInput, BBB, C, V))
absoluteVerseNumber = ourBibleOrganisationalSystem.getAbsoluteVerseNumber(
BBB, C, V)
vPrint(
'Quiet', debuggingThisModule,
_(" {} {}:{} is verse number {:,} in the {} versification system."
).format(BBB, C, V, absoluteVerseNumber,
ourVersificationSystem))
if BibleOrgSysGlobals.debugFlag:
vPrint(
'Quiet', debuggingThisModule,
_(" {} {}:{} is verse number 0x{:04x} in the {} versification system."
).format(BBB, C, V, absoluteVerseNumber,
ourVersificationSystem))
else:
vPrint(
'Quiet', debuggingThisModule,
_("Unable to find a valid single verse reference in your input: {!r}"
).format(userInput))
if division in ('Same','Different') and BibleOrgSysGlobals.verbosityLevel < 3 \
and not BibleOrgSysGlobals.debugFlag:
continue
vPrint('Quiet', debuggingThisModule,
" {} results are:".format(division))
for field, fResult in dResults.items():
vPrint('Quiet', debuggingThisModule,
" {}: {}".format(field, fResult))
# end of USFMBookCompare.main
if __name__ == '__main__':
multiprocessing.freeze_support(
) # Multiprocessing support for frozen Windows executables
demoFlag = False # Set to true to run the demo instead of main()
# Configure basic Bible Organisational System (BOS) set-up
parser = BibleOrgSysGlobals.setup(PROGRAM_NAME, PROGRAM_VERSION)
if not demoFlag:
parser.add_argument('file1', help="USFM Bible book file 1")
parser.add_argument('file2', help="USFM Bible book file 2")
BibleOrgSysGlobals.addStandardOptionsAndProcess(parser)
if demoFlag: demo()
else: main()
BibleOrgSysGlobals.closedown(PROGRAM_NAME, PROGRAM_VERSION)
# end of USFMBookCompare.py
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
def demoFile(name, filename, folder, BBB):
vPrint('Normal', debuggingThisModule,
_("Loading {} from {}…").format(BBB, filename))
EBB = ESFMBibleBook(name, BBB)
EBB.load(filename, folder)
vPrint('Normal', debuggingThisModule,
" ID is {!r}".format(EBB.getField('id')))
vPrint('Normal', debuggingThisModule,
" Header is {!r}".format(EBB.getField('h')))
vPrint(
'Normal', debuggingThisModule,
" Main titles are {!r} and {!r}".format(EBB.getField('mt1'),
EBB.getField('mt2')))
#vPrint( 'Quiet', debuggingThisModule, EBB )
EBB.validateMarkers()
EBBVersification = EBB.getVersification()
vPrint('Info', debuggingThisModule, EBBVersification)
UBBAddedUnits = EBB.getAddedUnits()
vPrint('Info', debuggingThisModule, UBBAddedUnits)
discoveryDict = EBB._discover()
#vPrint( 'Quiet', debuggingThisModule, "discoveryDict", discoveryDict )
EBB.check()
EBErrors = EBB.getCheckResults()
vPrint('Info', debuggingThisModule, EBErrors)
# end of fullDemoFile
from BibleOrgSys.InputOutput import USFMFilenames
if 1: # Test individual files
#name, testFolder, filename, BBB = "WEB", Path( '/mnt/SSDs/Bibles/English translations/WEB (World English Bible)/2012-06-23 eng-web_usfm/'), "06-JOS.usfm", "JOS" # You can put your test file here
#name, testFolder, filename, BBB = "WEB", Path( '/mnt/SSDs/Bibles/English translations/WEB (World English Bible)/2012-06-23 eng-web_usfm/'), "44-SIR.usfm", "SIR" # You can put your test file here
#name, testFolder, filename, BBB = "Matigsalug", Path( '/mnt/SSDs/Matigsalug/Bible/MBTV/'), "MBT102SA.SCP", "SA2" # You can put your test file here
#name, testFolder, filename, BBB = "Matigsalug", Path( '/mnt/SSDs/Matigsalug/Bible/MBTV/'), "MBT15EZR.SCP", "EZR" # You can put your test file here
name, testFolder, filename, BBB = "Matigsalug", Path(
'/mnt/SSDs/Matigsalug/Bible/MBTV/'
), "MBT41MAT.SCP", "MAT" # You can put your test file here
#name, testFolder, filename, BBB = "Matigsalug", Path( '/mnt/SSDs/Matigsalug/Bible/MBTV/'), "MBT67REV.SCP", "REV" # You can put your test file here
if os.access(testFolder, os.R_OK):
demoFile(name, filename, testFolder, BBB)
else:
vPrint(
'Quiet', debuggingThisModule,
f"Sorry, test folder '{testFolder}' doesn't exist on this computer."
)
if 1: # Test a whole folder full of files
name, testFolder = "Matigsalug", Path(
'/mnt/SSDs/Matigsalug/Bible/MBTV/'
) # You can put your test folder here
#name, testFolder = "WEB", Path( '/mnt/SSDs/Bibles/English translations/WEB (World English Bible)/2012-06-23 eng-web_usfm/' ) # You can put your test folder here
if os.access(testFolder, os.R_OK):
vPrint('Normal', debuggingThisModule,
_("Scanning {} from {}…").format(name, testFolder))
fileList = USFMFilenames.USFMFilenames(
testFolder).getMaximumPossibleFilenameTuples()
for BBB, filename in fileList:
demoFile(name, filename, testFolder, BBB)
else:
vPrint(
'Quiet', debuggingThisModule,
f"Sorry, test folder '{testFolder}' doesn't exist on this computer."
)
def briefDemo() -> None:
"""
Main program to handle command line parameters and then run what they want.
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
# Demo the languages object
lg = ISO_639_3_Languages().loadData(
) # Doesn't reload the XML unnecessarily :)
vPrint('Quiet', debuggingThisModule, lg) # Just print a summary
for testCode in (
'qwq',
'mbt',
'MBT',
'abk',
):
vPrint('Quiet', debuggingThisModule, " Testing {}…".format(testCode))
if not lg.isValidLanguageCode(testCode):
vPrint('Quiet', debuggingThisModule,
" {} not found".format(testCode))
else:
vPrint(
'Quiet', debuggingThisModule,
" {} -> {}".format(testCode, lg.getLanguageName(testCode)))
vPrint(
'Quiet', debuggingThisModule,
" Scope is {}, Type is {}".format(lg.getScope(testCode),
lg.getType(testCode)))
part1Code, part2Code = lg.getPart1Code(testCode), lg.getPart2Code(
testCode)
if part1Code is not None:
vPrint('Quiet', debuggingThisModule,
" Part1 code is {}".format(part1Code))
if part2Code is not None:
vPrint('Quiet', debuggingThisModule,
" Part2 code is {}".format(part2Code))
for testName in (
'English',
'German',
'Deutsch',
'French',
'Ayta, Abellen',
'Manobo, Matigsalug',
'Manobo',
'SomeName',
):
vPrint('Quiet', debuggingThisModule, " Testing {}…".format(testName))
code = lg.getLanguageCode(testName)
if code is None:
vPrint('Quiet', debuggingThisModule,
" {} not found".format(testName))
else:
vPrint('Quiet', debuggingThisModule,
" {} -> {}".format(testName, code))
for testNamePortion in (
'English',
'German',
'Deutsch',
'French',
'Ayta, Abellen',
'Manobo, Matigsalug',
'Manobo',
'SomeName',
):
vPrint('Quiet', debuggingThisModule,
" Testing {}…".format(testNamePortion))
matches = lg.getNameMatches(testNamePortion)
for match in matches:
vPrint(
'Quiet', debuggingThisModule,
" Found {} = {}".format(lg.getLanguageCode(match), match))
else:
vPrint('Quiet', debuggingThisModule,
" {} not found".format(testNamePortion))
def fullDemo() -> None:
"""
Full demo to check class is working
"""
from BibleOrgSys.Formats import SwordModules
BibleOrgSysGlobals.introduceProgram( __name__, programNameVersion, LAST_MODIFIED_DATE )
im = SwordInstallManager()
if 0 and __name__ == '__main__': im.isUserDisclaimerConfirmed()
else: im.setUserDisclaimerConfirmed()
if 1: # try refreshing one repository
getRepoName = 'NET Bible'
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Refresh {} repository…".format( getRepoName ) )
im.currentRepoName = getRepoName
im.currentInstallFolderpath = im.currentTempFolder
im.refreshRemoteSource()
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint( 'Quiet', debuggingThisModule, "{} modules: {}".format( len(im.availableModules), im.availableModules.keys() ) )
if BibleOrgSysGlobals.verbosityLevel > 2:
for modName in im.availableModules:
vPrint( 'Quiet', debuggingThisModule, " {}: {}".format( modName, im.availableModules[modName][0] ) )
if 1: # try installing and testing a module from the above repository
getModuleName = 'NETfree'
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Install {}…".format( getModuleName ) )
im.currentInstallFolderpath = 'TempSwInstMgrTestData/'
if im.installModule( getModuleName ):
confData = im.availableModules[getModuleName]
if isinstance( confData, tuple ): confName = confData[1]
elif isinstance( confData, list ): confName = confData[0][1]
swMC = SwordModules.SwordModuleConfiguration( confName, im.currentInstallFolderpath )
swMC.loadConf()
vPrint( 'Quiet', debuggingThisModule, swMC )
swM = SwordModules.SwordModule( swMC )
swM.loadBooks( inMemoryFlag=True )
vPrint( 'Verbose', debuggingThisModule, swM )
if not swM.SwordModuleConfiguration.locked: swM.test()
if 0: # try refreshing one repository
getRepoName = 'eBible'
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Refresh {} repository…".format( getRepoName ) )
im.currentRepoName = getRepoName
im.currentInstallFolderpath = im.currentTempFolder
im.refreshRemoteSource()
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint( 'Quiet', debuggingThisModule, "{} modules: {}".format( len(im.availableModules), im.availableModules.keys() ) )
if BibleOrgSysGlobals.verbosityLevel > 2:
for modName in im.availableModules:
vPrint( 'Quiet', debuggingThisModule, " {}: {}".format( modName, im.availableModules[modName][0] ) )
if 1: # try installing and testing a module from the above repository
getModuleName = 'engWEBBE2015eb'
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Install {}…".format( getModuleName ) )
im.currentInstallFolderpath = 'TempSwInstMgrTestData/'
if im.installModule( getModuleName ):
swMC = SwordModules.SwordModuleConfiguration( getModuleName, im.currentInstallFolderpath )
swMC.loadConf()
vPrint( 'Quiet', debuggingThisModule, swMC )
swM = SwordModules.SwordModule( swMC )
swM.loadBooks( inMemoryFlag=True )
vPrint( 'Verbose', debuggingThisModule, swM )
if not swM.SwordModuleConfiguration.locked: swM.test()
if 0: # try refreshing all repositories
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Refresh all repositories…" )
im.refreshAllRemoteSources()
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint( 'Quiet', debuggingThisModule, "{} modules: {}".format( len(im.availableModules), im.availableModules.keys() ) )
for modName in im.availableModules:
vPrint( 'Quiet', debuggingThisModule, " {}: {}".format( modName, im.availableModules[modName][0] ) )
if 1: # try installing another module
getModuleName = 'JPS'
vPrint( 'Quiet', debuggingThisModule, "\nDemo: Install {}…".format( getModuleName ) )
im.currentRepoName = 'CrossWire Main'
im.currentInstallFolderpath = 'TempSwInstMgrTestData/'
if im.installModule( getModuleName ): # See if we can read it
confData = im.availableModules[getModuleName]
if isinstance( confData, tuple ): confName = confData[1]
elif isinstance( confData, list ): confName = confData[0][1]
swMC = SwordModules.SwordModuleConfiguration( confName, im.currentInstallFolderpath )
swMC.loadConf()
vPrint( 'Quiet', debuggingThisModule, swMC )
swM = SwordModules.SwordModule( swMC )
swM.loadBooks( inMemoryFlag=True )
vPrint( 'Verbose', debuggingThisModule, swM )
if not swM.SwordModuleConfiguration.locked: swM.test()
def __init__(self,
parameterOne,
resourcesObject=None,
downloadAllBooks=False) -> None:
"""
Create the Door43 cataloged Bible object.
parameterOne can be:
a catalog dictionary entry (and second parameter must be None)
or
an index into the BibleList in the resourcesObject passed as the second parameter
"""
fnPrint(
debuggingThisModule,
f"DCSBible.__init__( {parameterOne}, {resourcesObject}, {downloadAllBooks} )…"
)
if isinstance(parameterOne, dict):
assert resourcesObject is None
resourceDict = parameterOne
else:
assert isinstance(parameterOne, int)
assert resourcesObject # why ??? and isinstance( resourcesObject, Door43CatalogResources )
resourceDict = resourcesObject.getBibleResourceDict(parameterOne)
assert resourceDict and isinstance(resourceDict, dict)
#dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict )
#dPrint( 'Quiet', debuggingThisModule, 'resourceDict', resourceDict.keys() )
self.baseURL = resourceDict['html_url']
#dPrint( 'Quiet', debuggingThisModule, 'self.baseURL', self.baseURL )
adjustedRepoName = resourceDict['full_name'].replace('/', '--')
#dPrint( 'Quiet', debuggingThisModule, 'adjustedRepoName', adjustedRepoName )
desiredFolderName = BibleOrgSysGlobals.makeSafeFilename(
adjustedRepoName)
unzippedFolderpath = DEFAULT_DOWNLOAD_FOLDERPATH.joinpath(
f'{adjustedRepoName}/')
if downloadAllBooks:
# See if files already exist and are current (so don't download again)
alreadyDownloadedFlag = False
if os.path.isdir(unzippedFolderpath):
#dPrint( 'Quiet', debuggingThisModule, f"Issued: {resourceDict['issued']}" )
updatedDatetime = datetime.strptime(resourceDict['updated_at'],
'%Y-%m-%dT%H:%M:%SZ')
#dPrint( 'Quiet', debuggingThisModule, f"updatedDatetime: {updatedDatetime}" )
#dPrint( 'Quiet', debuggingThisModule, f"folder: {os.stat(unzippedFolderpath).st_mtime}" )
folderModifiedDatetime = datetime.fromtimestamp(
os.stat(unzippedFolderpath).st_mtime)
#dPrint( 'Quiet', debuggingThisModule, f"folderModifiedDatetime: {folderModifiedDatetime}" )
alreadyDownloadedFlag = folderModifiedDatetime > updatedDatetime
#dPrint( 'Quiet', debuggingThisModule, f"alreadyDownloadedFlag: {alreadyDownloadedFlag}" )
if alreadyDownloadedFlag:
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint(
'Quiet', debuggingThisModule,
"Skipping download because folder '{}' already exists."
.format(unzippedFolderpath))
else: # Download the zip file (containing all the USFM files, README.md, LICENSE.md, manifest.yaml, etc.)
# TODO: Change to .tar.gz instead of zip
zipURL = self.baseURL + '/archive/master.zip' # '/archive/master.tar.gz'
if BibleOrgSysGlobals.verbosityLevel > 1:
vPrint('Quiet', debuggingThisModule,
"Downloading entire repo from '{}'…".format(zipURL))
try:
HTTPResponseObject = urllib.request.urlopen(zipURL)
except urllib.error.URLError as err:
#errorClass, exceptionInstance, traceback = sys.exc_info()
#dPrint( 'Quiet', debuggingThisModule, '{!r} {!r} {!r}'.format( errorClass, exceptionInstance, traceback ) )
logging.critical("DCS URLError '{}' from {}".format(
err, zipURL))
return
#dPrint( 'Quiet', debuggingThisModule, " HTTPResponseObject", HTTPResponseObject )
contentType = HTTPResponseObject.info().get('content-type')
if BibleOrgSysGlobals.debugFlag and debuggingThisModule:
vPrint('Quiet', debuggingThisModule, " contentType",
repr(contentType))
if contentType == 'application/octet-stream':
try:
os.makedirs(unzippedFolderpath)
except FileExistsError:
pass
downloadedData = HTTPResponseObject.read()
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint(
'Quiet', debuggingThisModule,
f" Downloaded {len(downloadedData):,} bytes from '{zipURL}'"
)
# Bug in Python up to 3.7 makes this not work for large aligned Bibles (3+ MB)
# myTempFile = tempfile.SpooledTemporaryFile()
myTempFile = tempfile.TemporaryFile()
myTempFile.write(downloadedData)
with zipfile.ZipFile(myTempFile) as myzip:
# NOTE: Could be a security risk here
myzip.extractall(unzippedFolderpath)
myTempFile.close() # Automatically deletes the file
else:
vPrint('Quiet', debuggingThisModule, " contentType",
repr(contentType))
halt # unknown content type
self.downloadedAllBooks = True
# There's probably a folder inside this folder
folders = os.listdir(unzippedFolderpath)
#dPrint( 'Quiet', debuggingThisModule, 'folders', folders )
assert len(
folders
) == 1 # else maybe a previous download failed -- just manually delete the folder
desiredFolderName = folders[0] + '/'
#dPrint( 'Quiet', debuggingThisModule, 'desiredFolderName', desiredFolderName )
USFMBible.__init__(self,
os.path.join(unzippedFolderpath,
desiredFolderName),
givenName=resourceDict['name'])
else: # didn't request all books to be downloaded at once
self.downloadedAllBooks = False
self.attemptedDownload = {}
try:
os.makedirs(unzippedFolderpath)
except FileExistsError:
pass
USFMBible.__init__(self,
unzippedFolderpath,
givenName=resourceDict['name'])
self.objectNameString = 'DCS USFM Bible object'
self.uWencoded = True
def __load(self, XMLFileOrFilepath):
"""
Load the source XML file and remove the header from the tree.
Also, extracts some useful elements from the header element.
"""
assert XMLFileOrFilepath
self.__XMLFileOrFilepath = XMLFileOrFilepath
assert self._XMLTree is None or len(
self._XMLTree) == 0 # Make sure we're not doing this twice
vPrint(
'Info', debuggingThisModule,
_("Loading USFM3Markers XML file from {!r}…").format(
self.__XMLFileOrFilepath))
self._XMLTree = ElementTree().parse(self.__XMLFileOrFilepath)
assert self._XMLTree # Fail here if we didn't load anything at all
if self._XMLTree.tag == self._treeTag:
header = self._XMLTree[0]
if header.tag == self._headerTag:
self.XMLheader = header
self._XMLTree.remove(header)
BibleOrgSysGlobals.checkXMLNoText(header, 'header')
BibleOrgSysGlobals.checkXMLNoTail(header, 'header')
BibleOrgSysGlobals.checkXMLNoAttributes(header, 'header')
if len(header) > 1:
logging.info(_("Unexpected elements in header"))
elif len(header) == 0:
logging.info(_("Missing work element in header"))
else:
work = header[0]
BibleOrgSysGlobals.checkXMLNoText(work, "work in header")
BibleOrgSysGlobals.checkXMLNoTail(work, "work in header")
BibleOrgSysGlobals.checkXMLNoAttributes(
work, "work in header")
if work.tag == "work":
self.PROGRAM_VERSION = work.find('version').text
self.dateString = work.find('date').text
self.titleString = work.find('title').text
else:
logging.warning(_("Missing work element in header"))
else:
logging.warning(
_("Missing header element (looking for {!r} tag)".format(
self._headerTag)))
if header.tail is not None and header.tail.strip():
logging.error(
_("Unexpected {!r} tail data after header").format(
element.tail))
else:
logging.error(
_("Expected to load {!r} but got {!r}").format(
self._treeTag, self._XMLTree.tag))
def fullDemo() -> None:
"""
Full demo to check class is working
"""
from BibleOrgSys.Reference.VerseReferences import SimpleVerseKey
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
# Test the DCSBibles class (also used later)
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint('Quiet', debuggingThisModule, "\n\nA/ DCSBibles class test…")
dcsBibles = DCSBibles()
vPrint('Quiet', debuggingThisModule, dcsBibles, end='\n\n')
#dcsBibles.load() # takes a minute
#dPrint( 'Quiet', debuggingThisModule, dcsBibles )
dcsBibles.fetchAllBibles()
if 0: # print the list
vPrint('Quiet', debuggingThisModule,
"Bible list ({}):".format(len(dcsBibles.BibleList)))
for j, BibleDict in enumerate(dcsBibles.BibleList, start=1):
ownerName = BibleDict['owner']['full_name']
if not ownerName: ownerName = BibleDict['owner']['username']
vPrint('Normal', debuggingThisModule,
f" Entry {j:3} '{BibleDict['name']}' '{ownerName}'")
testRefs = (
('GEN', '1', '1'),
('GEN', '2', '2'),
('JER', '33', '3'),
('MAL', '4', '6'),
('MAT', '1', '1'),
('JHN', '3', '16'),
('JDE', '1', '14'),
('REV', '22', '21'),
)
def processSearchResult(searchResult: dict,
downloadAllBooks: bool = False) -> None:
if searchResult and isinstance(searchResult, dict):
dcsBible1 = DCSBible(searchResult,
downloadAllBooks=downloadAllBooks)
try:
dcsBible1.preload()
except FileNotFoundError:
assert downloadAllBooks == False
vPrint('Normal', debuggingThisModule, dcsBible1, end='\n\n')
for testRef in testRefs:
verseKey = SimpleVerseKey(*testRef)
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint('Quiet', debuggingThisModule, verseKey)
vPrint('Quiet', debuggingThisModule, " ",
dcsBible1.getVerseDataList(verseKey))
elif BibleOrgSysGlobals.verbosityLevel > 0:
vPrint('Quiet', debuggingThisModule,
f"Unexpected search result: {searchResult}")
# end of processSearchResult function
if 1: # Test the DCSBible class with the ULT
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint('Quiet', debuggingThisModule, "\n\nB/ ULT test")
downloadAllBooks = True
searchResult = dcsBibles.searchReposExact('unfoldingWord', 'en_ult')
if searchResult:
#dPrint( 'Quiet', debuggingThisModule, 'searchResult', type(searchResult), len(searchResult), searchResult )
if isinstance(searchResult, dict):
processSearchResult(searchResult, downloadAllBooks)
elif isinstance(searchResult, list):
vPrint('Quiet', debuggingThisModule,
f"Found {len(searchResult)} 'en_ult' repos!")
searchResults = searchResult
for searchResult in searchResults:
processSearchResult(searchResult, downloadAllBooks)
else:
logging.critical(
f"Bad search result {type(searchResult)} ({len(searchResult)}): {searchResult}"
)
else:
logging.critical(f"Empty search result: {searchResult}")
if 1: # Test the DCSBible class with the UST
if BibleOrgSysGlobals.verbosityLevel > 0:
vPrint('Quiet', debuggingThisModule, "\n\nC/ UST test")
downloadAllBooks = False
searchResult = dcsBibles.searchReposExact('unfoldingWord', 'en_ust')
if searchResult:
#dPrint( 'Quiet', debuggingThisModule, 'searchResult', type(searchResult), len(searchResult), searchResult )
if isinstance(searchResult, dict):
processSearchResult(searchResult, downloadAllBooks)
elif isinstance(searchResult, list):
vPrint('Quiet', debuggingThisModule,
f"Found {len(searchResult)} 'en_ust' repos!")
searchResults = searchResult
for searchResult in searchResults:
processSearchResult(searchResult, downloadAllBooks)
else:
logging.critical(
f"Bad search result {type(searchResult)} ({len(searchResult)}): {searchResult}"
)
else:
logging.critical(f"Empty search result: {searchResult}")
def fullDemo() -> None:
"""
Full demo to check class is working
"""
BibleOrgSysGlobals.introduceProgram(__name__, programNameVersion,
LAST_MODIFIED_DATE)
# if 0: # demonstrate the Bible Lexicon Index class
# vPrint( 'Normal', debuggingThisModule, "\nDemonstrating the Bible Lexicon Index class…" )
# blix = BibleLexiconIndex() # Load and process the XML
# vPrint( 'Quiet', debuggingThisModule, blix ) # Just print a summary
# vPrint( 'Quiet', debuggingThisModule, '' )
# vPrint( 'Quiet', debuggingThisModule, "Code for 2 is", blix.getLexiconCodeFromStrongsNumber( '2' ) )
# vPrint( 'Quiet', debuggingThisModule, "Code for H8674 is", blix.getLexiconCodeFromStrongsNumber( 'H8674' ) )
# vPrint( 'Quiet', debuggingThisModule, "Code for H8675 is", blix.getLexiconCodeFromStrongsNumber( 'H8675' ) )
# vPrint( 'Quiet', debuggingThisModule, "Codes for aac are", blix.getStrongsNumberFromLexiconCode('aac'), blix.getBrDrBrCodeFromLexiconCode('aac'), blix.getTWOTCodeFromLexiconCode('aac') )
# vPrint( 'Quiet', debuggingThisModule, "Codes for nyy are", blix.getStrongsNumberFromLexiconCode('nyy'), blix.getBrDrBrCodeFromLexiconCode('nyy'), blix.getTWOTCodeFromLexiconCode('nyy') )
# vPrint( 'Quiet', debuggingThisModule, "Codes for pdc are", blix.getStrongsNumberFromLexiconCode('pdc'), blix.getBrDrBrCodeFromLexiconCode('pdc'), blix.getTWOTCodeFromLexiconCode('pdc') )
# vPrint( 'Quiet', debuggingThisModule, "Codes for pdd are", blix.getStrongsNumberFromLexiconCode('pdd'), blix.getBrDrBrCodeFromLexiconCode('pdd'), blix.getTWOTCodeFromLexiconCode('pdd') )
if 1: # demonstrate the Bible Lexicon class
vPrint('Normal', debuggingThisModule,
"\nDemonstrating the Bible Lexicon class…")
bl = BibleLexicon() # Load and process the XML
vPrint('Quiet', debuggingThisModule, bl) # Just print a summary
vPrint('Quiet', debuggingThisModule, '')
for strongsKey in (
'H1',
'H123',
'H165',
'H1732',
'H1979',
'H2011',
'H8674',
'H8675',
'G1',
'G123',
'G165',
'G1732',
'G1979',
'G2011',
'G5624',
'G5625',
): # Last ones of each are invalid
vPrint('Quiet', debuggingThisModule, '\n' + strongsKey)
vPrint('Quiet', debuggingThisModule, " Data (all):",
bl.getStrongsEntryData(strongsKey))
vPrint('Quiet', debuggingThisModule, " 'Usage' field:",
bl.getStrongsEntryField(strongsKey, 'usage'))
vPrint('Quiet', debuggingThisModule, " HTML entry:",
bl.getStrongsEntryHTML(strongsKey))
for BrDrBrKey in (
'a.ab.ac',
'a.gq.ab',
'b.aa.aa',
'xw.ah.ah',
'xy.zz.zz',
): # Last one is invalid
vPrint('Quiet', debuggingThisModule, '\n' + BrDrBrKey)
vPrint('Quiet', debuggingThisModule, " Data (all):",
bl.getBrDrBrEntryData(BrDrBrKey))
vPrint('Quiet', debuggingThisModule, " 'Status' field:",
bl.getBrDrBrEntryField(BrDrBrKey, 'status'))
vPrint('Quiet', debuggingThisModule, " HTML entry:",
bl.getBrDrBrEntryHTML(BrDrBrKey))
def __validateSystem( self, bookOrderTree, systemName ):
""" Do a semi-automatic check of the XML file validity. """
assert bookOrderTree
uniqueDict = {}
for elementName in self.uniqueElements: uniqueDict["Element_"+elementName] = []
for attributeName in self.uniqueAttributes: uniqueDict["Attribute_"+attributeName] = []
expectedID = 1
for k,element in enumerate(bookOrderTree):
if element.tag == self.mainElementTag:
BibleOrgSysGlobals.checkXMLNoTail( element, element.tag )
if not self.compulsoryAttributes and not self.optionalAttributes: BibleOrgSysGlobals.checkXMLNoAttributes( element, element.tag )
if not self.compulsoryElements and not self.optionalElements: BibleOrgSysGlobals.checkXMLNoSubelements( element, element.tag )
# Check ascending ID field
ID = element.get("id")
intID = int( ID )
if intID != expectedID:
logging.error( _("ID numbers out of sequence in record {} (got {} when expecting {}) for {}").format( k, intID, expectedID, systemName ) )
expectedID += 1
# Check that this is unique
if element.text:
if element.text in uniqueDict:
logging.error( _("Found {!r} data repeated in {!r} element in record with ID {!r} (record {}) for {}").format( element.text, element.tag, ID, k, systemName ) )
uniqueDict[element.text] = None
# Check compulsory attributes on this main element
for attributeName in self.compulsoryAttributes:
attributeValue = element.get( attributeName )
if attributeValue is None:
logging.error( _("Compulsory {!r} attribute is missing from {} element in record {}").format( attributeName, element.tag, k ) )
if not attributeValue:
logging.warning( _("Compulsory {!r} attribute is blank on {} element in record {}").format( attributeName, element.tag, k ) )
# Check optional attributes on this main element
for attributeName in self.optionalAttributes:
attributeValue = element.get( attributeName )
if attributeValue is not None:
if not attributeValue:
logging.warning( _("Optional {!r} attribute is blank on {} element in record {}").format( attributeName, element.tag, k ) )
# Check for unexpected additional attributes on this main element
for attributeName in element.keys():
attributeValue = element.get( attributeName )
if attributeName not in self.compulsoryAttributes and attributeName not in self.optionalAttributes:
logging.warning( _("Additional {!r} attribute ({!r}) found on {} element in record {}").format( attributeName, attributeValue, element.tag, k ) )
# Check the attributes that must contain unique information (in that particular field -- doesn't check across different attributes)
for attributeName in self.uniqueAttributes:
attributeValue = element.get( attributeName )
if attributeValue is not None:
if attributeValue in uniqueDict["Attribute_"+attributeName]:
logging.error( _("Found {!r} data repeated in {!r} field on {} element in record {}").format( attributeValue, attributeName, element.tag, k ) )
uniqueDict["Attribute_"+attributeName].append( attributeValue )
# Check compulsory elements
for elementName in self.compulsoryElements:
if element.find( elementName ) is None:
logging.error( _("Compulsory {!r} element is missing in record with ID {!r} (record {})").format( elementName, ID, k ) )
if not element.find( elementName ).text:
logging.warning( _("Compulsory {!r} element is blank in record with ID {!r} (record {})").format( elementName, ID, k ) )
# Check optional elements
for elementName in self.optionalElements:
if element.find( elementName ) is not None:
if not element.find( elementName ).text:
logging.warning( _("Optional {!r} element is blank in record with ID {!r} (record {})").format( elementName, ID, k ) )
# Check for unexpected additional elements
for subelement in element:
if subelement.tag not in self.compulsoryElements and subelement.tag not in self.optionalElements:
logging.warning( _("Additional {!r} element ({!r}) found in record with ID {!r} (record {})").format( subelement.tag, subelement.text, ID, k ) )
# Check the elements that must contain unique information (in that particular element -- doesn't check across different elements)
for elementName in self.uniqueElements:
if element.find( elementName ) is not None:
text = element.find( elementName ).text
if text in uniqueDict["Element_"+elementName]:
logging.error( _("Found {!r} data repeated in {!r} element in record with ID {!r} (record {})").format( text, elementName, ID, k ) )
uniqueDict["Element_"+elementName].append( text )
else:
logging.warning( _("Unexpected element: {} in record {}").format( element.tag, k ) )
def _validateSystem(self, punctuationTree, systemName):
"""
"""
assert punctuationTree
uniqueDict = {}
for elementName in self.uniqueElements:
uniqueDict["Element_" + elementName] = []
for attributeName in self.uniqueAttributes:
uniqueDict["Attribute_" + attributeName] = []
for k, element in enumerate(punctuationTree):
if element.tag in self.mainElementTags:
BibleOrgSysGlobals.checkXMLNoTail(element, element.tag)
if not self.compulsoryAttributes and not self.optionalAttributes:
BibleOrgSysGlobals.checkXMLNoAttributes(
element, element.tag)
if not self.compulsoryElements and not self.optionalElements:
BibleOrgSysGlobals.checkXMLNoSubelements(
element, element.tag)
# Check compulsory attributes on this main element
for attributeName in self.compulsoryAttributes:
attributeValue = element.get(attributeName)
if attributeValue is None:
logging.error(
_("Compulsory {!r} attribute is missing from {} element in record {}"
).format(attributeName, element.tag, k))
if not attributeValue:
logging.warning(
_("Compulsory {!r} attribute is blank on {} element in record {}"
).format(attributeName, element.tag, k))
# Check optional attributes on this main element
for attributeName in self.optionalAttributes:
attributeValue = element.get(attributeName)
if attributeValue is not None:
if not attributeValue:
logging.warning(
_("Optional {!r} attribute is blank on {} element in record {}"
).format(attributeName, element.tag, k))
# Check for unexpected additional attributes on this main element
for attributeName in element.keys():
attributeValue = element.get(attributeName)
if attributeName not in self.compulsoryAttributes and attributeName not in self.optionalAttributes:
logging.warning(
_("Additional {!r} attribute ({!r}) found on {} element in record {}"
).format(attributeName, attributeValue,
element.tag, k))
# Check the attributes that must contain unique information (in that particular field -- doesn't check across different attributes)
for attributeName in self.uniqueAttributes:
attributeValue = element.get(attributeName)
if attributeValue is not None:
if attributeValue in uniqueDict["Attribute_" +
attributeName]:
logging.error(
_("Found {!r} data repeated in {!r} field on {} element in record {}"
).format(attributeValue, attributeName,
element.tag, k))
uniqueDict["Attribute_" +
attributeName].append(attributeValue)
# Check compulsory elements
for elementName in self.compulsoryElements:
if element.find(elementName) is None:
logging.error(
_("Compulsory {!r} element is missing in record with ID {!r} (record {})"
).format(elementName, ID, k))
if not element.find(elementName).text:
logging.warning(
_("Compulsory {!r} element is blank in record with ID {!r} (record {})"
).format(elementName, ID, k))
# Check optional elements
for elementName in self.optionalElements:
if element.find(elementName) is not None:
if not element.find(elementName).text:
logging.warning(
_("Optional {!r} element is blank in record with ID {!r} (record {})"
).format(elementName, ID, k))
# Check for unexpected additional elements
for subelement in element:
if subelement.tag not in self.compulsoryElements and subelement.tag not in self.optionalElements:
logging.warning(
_("Additional {!r} element ({!r}) found in record with ID {!r} (record {})"
).format(subelement.tag, subelement.text, ID, k))
# Check the elements that must contain unique information (in that particular element -- doesn't check across different elements)
for elementName in self.uniqueElements:
if element.find(elementName) is not None:
text = element.find(elementName).text
if text in uniqueDict["Element_" + elementName]:
logging.error(
_("Found {!r} data repeated in {!r} element in record with ID {!r} (record {})"
).format(text, elementName, ID, k))
uniqueDict["Element_" + elementName].append(text)
else:
logging.warning(
_("Unexpected element: {} in record {}").format(
element.tag, k))
def test_applyStringAdjustments(self):
longText = "The quick brown fox jumped over the lazy brown dog."
adjustments = [(36,'lazy','fat'),(0,'The','A'),(20,'jumped','tripped'),(4,'','very '),(10,'brown','orange')]
result = BibleOrgSysGlobals.applyStringAdjustments( longText, adjustments )
self.assertEqual( result, "A very quick orange fox tripped over the fat brown dog." )
umc.pickle() # Produce a pickle output file
umc.exportDataToJSON() # Produce a json output file
umc.exportDataToPython() # Produce the .py tables
# umc.exportDataToC() # Produce the .h and .c tables
else: # Must be demo mode
# Demo the converter object
umc = USFM2MarkersConverter().loadAndValidate() # Load the XML
vPrint( 'Quiet', debuggingThisModule, umc ) # Just print a summary
# end of fullDemo
def fullDemo() -> None:
"""
Full demo to check class is working
"""
briefDemo()
# end of fullDemo
if __name__ == '__main__':
from multiprocessing import freeze_support
freeze_support() # Multiprocessing support for frozen Windows executables
# Configure basic Bible Organisational System (BOS) set-up
parser = BibleOrgSysGlobals.setup( SHORT_PROGRAM_NAME, PROGRAM_VERSION, LAST_MODIFIED_DATE )
BibleOrgSysGlobals.addStandardOptionsAndProcess( parser, exportAvailable=True )
fullDemo()
BibleOrgSysGlobals.closedown( PROGRAM_NAME, PROGRAM_VERSION )
# end of USFM2MarkersConverter.py
def setUp( self ):
parser = BibleOrgSysGlobals.setup( PROGRAM_NAME, PROGRAM_VERSION, LAST_MODIFIED_DATE )
# BibleOrgSysGlobals.addStandardOptionsAndProcess( parser )
BibleOrgSysGlobals.preloadCommonData()
# Create the BibleOrganisationalSystemsConverter object
self.bossc = BibleOrganisationalSystemsConverter.BibleOrganisationalSystemsConverter().loadAndValidate() # Doesn't reload the XML unnecessarily :)
