def closeOutSessionLog(self):
'''This will write out all the logged entries to the log files at the
end of a process. However, we will first check to see if we are in a
valid project area. If not, it all comes to a crashing halt.'''
if tools.isProjectFolder() == True:
# First we need to be sure there is a Log folder to write to
if os.path.isdir(self._logFolder) == False:
os.mkdir(self._logFolder)
# Close process log
if os.path.isfile(self._processLogFile) == True:
processWriteObject = codecs.open(self._processLogFile,
"a",
encoding='utf_8')
else:
processWriteObject = codecs.open(self._processLogFile,
"w",
encoding='utf_8')
for line in self._processLogObject:
processWriteObject.write(line)
processWriteObject.close()
self._processLogObject = []
else:
tools.userMessage(
"Sorry, cannot write out the log files because we do not seem to be inside a project."
)
def main (self, templateName) :
'''Here we will manage the template making process.'''
self.settings = tools.getSettingsObject()
# For the location we use whatever the makefile.conf file has
# whether it is abs or relative. Note, we use abs in archive_project.py
templateFilePath = self.settings['Process']['Paths']['PATH_TEMPLATES']
templateFile = templateFilePath + "/" + templateName + ".tar.gz"
# Let's look to see if the Template folder is there
if not os.path.isdir(templateFilePath) :
os.mkdir(templateFilePath)
tar = tarfile.open(templateFile, 'w:gz')
# We only want to add project settings files to the template so
# we will only take the files in the root of the project (all
# except the Makefile file).
# how do we do this??????????????????
tar.add(???)
tar.close()
# Tell the world what we did
tools.userMessage('A template has been made from the current project settings')
def main(self, projType, newFolderName):
'''Create a new project at the specified path.'''
# Just in case it isn't already a full path
newProjectPath = os.path.abspath(newFolderName)
fileLib = os.environ.get('PTXPLUS_BASE') + "/resources/lib_sysFiles"
# Check to see if we support this type of publication
if projType in tools.getSystemSettingsObject(
)['System']['pubTypeList'] and projType != 'dictionary':
# To make a project all we really need is the .conf file
if projType in tools.getSystemSettingsObject(
)['System']['pubTypeList']:
if not os.access(newProjectPath + "/." + projType + ".conf",
os.R_OK):
shutil.copy(fileLib + "/." + projType + ".conf",
newProjectPath + "/." + projType + ".conf")
# Now to give the user a clue as to what happened we will
# write out a little new project readme file with enough
# info to guide them on to the next step.
if not os.path.isfile(newProjectPath + "/README"):
shutil.copy(fileLib + "/README", newProjectPath + "/README")
else:
userMessage("ERRR: The project type: [" + projType +
"] is unknown. Process halted!")
sys.exit(1)
tools.userMessage('INFO: Created new project at: ' +
newProjectPath)
else:
tools.userMessage('ERRR: The [' + projType +
'] publication type is not supported.')
def pruneDirs(self, root, dirs):
''' Remove all unwanted directories from the dirs list in place.'''
try:
excludeDirs = self.settings['General']['Archive'][
'excludeArchiveDirs'].split()
c = 0
while c < len(dirs):
if dirs[c] in (excludeDirs):
del dirs[c]
else:
c = c + 1
except:
tools.userMessage(
'No excludeArchiveDirs list found in project.conf')
def main(self):
'''This is the main function for restoring a project from a backup.
We assume that we are starting from inside the project folder.'''
settings = tools.getSettingsObject()
# For the location we use whatever the makefile.conf file has
# whether it is abs or relative. Note, we use abs in archive_project.py
backupFilePath = settings['General']['Backup']['backupPath']
backupFile = backupFilePath + "/Backup.tar.gz"
tar = tarfile.open(backupFile, 'r:gz')
tar.extractall()
tar.close()
# Tell the world what we did
tools.userMessage("Restore project completed.")
def main(self, task, typeID, inputFile, outputFile,
optionalPassedVariable):
'''This is the main routine for the class. It will control
the running of the process classes we want to run.'''
# Set some global (might be better done in an init section)
self._task = task
self._typeID = typeID
self._inputFile = inputFile
self._outputFile = outputFile
self._optionalPassedVariable = optionalPassedVariable
# We need to sort out the task that we are running
# Sometimes parent meta-tasks are being called which
# need to link to the individual tasks. This sorts that
# out and runs everthing that is called to run.
# Make a list that contains all the metaProcesses
metaTaskList = []
taskList = []
metaTaskList = log_manager._settings['System']['Processes'][
'textMetaProcesses']
# if this is a meta task then we need to process it as
# if there are multiple sub-tasks within even though
# there may only be one
if self._task in metaTaskList:
metaTask = self._task
taskList = log_manager._settings['System']['Processes'][metaTask]
for thisTask in taskList:
# It would be good if we gave a little feedback to the user
# as to what exactly which processes are going to be run and
# on what.
head, tail = os.path.split(self._inputFile)
tools.userMessage('INFO: Now running: ' + thisTask + ' (' +
tail + ')')
# The standard sys.argv[1] setting contains the name of the metaTask
# However, that is not the name of the actual module we want to send
# off to process. We need to replace sys.argv[1] with the right task
# name and any parameters that go with it.
sys.argv[1] = thisTask
self.runIt(thisTask)
# If it is not a meta task then it must be a single one
# so we will just run it as it comes in
else:
self.runIt(self._task)
def recordError(self, event):
'''Record an error report line to the error log object.'''
# Check for the Log folder and make it if it isn't there.
if not os.path.isdir(self._logFolder):
os.mkdir(self._logFolder)
tools.userMessage("INFO: Created Log folder")
if os.path.isfile(self._errorLogFile) == True:
errorWriteObject = codecs.open(self._errorLogFile,
"a",
encoding='utf_8')
else:
errorWriteObject = codecs.open(self._errorLogFile,
"w",
encoding='utf_8')
errorWriteObject.write(event + '\n')
errorWriteObject.close()
def main(self, archiveFile):
# Look to see if the archiveFile exists, if not, we stop here
if os.path.isfile(archiveFile) == False:
tools.userMessage(
'The archive: ' + archiveFile +
' does not exist. Sorry can\'t import the project')
else:
tar = tarfile.open(archiveFile, 'r:gz')
# if directory already exists, bail out
if tools.isProjectFolder() == True:
tools.userMessage(
'There is already a project in this folder. Aborting import'
)
return
# Extract the tar file
tar.extractall()
tar.close()
# Create the project.conf file from the archive.conf file
shutil.move("archive.conf", "project.conf")
# There may be additional folders and fles to add to make the
# project complete. This depends on what version of ptxplus
# was used to create the project. The project will be automatically
# updated the first time it is run.
# Tell the world what we did
tools.userMessage('Import project complete')
def log(self, entryType, event, toTerm='false'):
'''This will be a simple ini type output the value can be
parsed as simple CVS. The output params are:
entryType = ERRR, WARN, INFO
context = A chunk of text which triggered the log event
event = A brief description of the log event'''
# Our version of a UID
entryID = tools.makeUID()
# Add sub process if there is one
if self._currentSubProcess:
event = self._currentSubProcess + ": " + event
# for consistant quoting (I think)
newEvent = event.replace('"', '""')
context = self._currentContext.replace('"', '""')
# Assemble the CSV entry line
entry = '"' + entryID + '","' + entryType + '","' + context + '","' + \
self._currentLocation + '","' + newEvent + '"'
#Collect the entry
self._processLogObject.append(entry + "\n")
# If for some reason we fail to find a debugMode setting we will default
# to debug output
try:
if self._settings != None and self._settings['System'][
'ErrorHandling']['Python']['debugMode'] == 'true':
tools.userMessage(entryType + ": " + event)
except:
tools.userMessage(entryType + ": " + event)
# To save on code we will give the option to output this same message to
# the termina which could be very handy in many cases
if toTerm.lower() == 'true':
tools.userMessage(entryType + ": " + event)
# Because there are so many ways to create errors we need just a simple
# way to track them across seperate processes. This will be done with a
# simple error.log. Each time an error is found it will be added to the
# object. The object will be written out at the end of the process.
# The error.log file will continue to colect error repors until the
# series of processes are done then the system will collect them all and
# report.
if entryType == "ERRR" or entryType == "WARN":
errorID = self._logProcessID + "." + entryID
errorEntry = '"' + errorID + '","' + entryType + '","' + \
self._currentLocation + '","' + context + '","' + newEvent + '"'
error_manager.recordError(errorEntry)
# Don't forget to count the errors
if entryType == "ERRR":
self._errorCount += 1
elif entryType == "WARN":
self._warningCount += 1
def outputProcessErrorReport(self):
'''Take a look at the error log and tell how many warnings and errors found.'''
errrOutput = ""
warnOutput = ""
errrCount = 0
warnCount = 0
if os.path.isfile(self._errorLogFile) == True:
fileObject = codecs.open(self._errorLogFile, "r", encoding='utf_8')
errrOutput = "\nErrors found: \n"
warnOutput = "\nWarnings found: \n"
for line in fileObject:
if line.find("ERRR") > 0:
errrCount += 1
elif line.find("WARN") > 0:
warnCount += 1
if errrCount > 0:
tools.userMessage("ERRR: A total of " + str(errrCount) +
" errors were found.")
# Now we will add a more "in your face" error report so they are not ignored
# It might be good to exchange the sed command for stanard Python regex code
try:
sed_filter = """sed -r 's/[[:blank:]]*("[^"]*")[[:blank:]]*,/\\1\\n/g' < Log/{log!r}"""\
"""| sed -r 's/(^[[:blank:]]*"|"[[:blank:]]*$)//g'""".format
dialog_command = "zenity --title={title!r} "\
"--window-icon={path!r}/resources/icons/ptxplus.png "\
"--height=400 --width=600 --list "\
"--text={text!r} "\
"--column='File' --column='Type' --column='Ref' --column='Context' --column='Description' "\
"--hide-column=1,2".format
os.system(
sed_filter(log='error.log') + ' | ' +
dialog_command(title='PtxPlus error log report',
path=basePath,
text='A total of {0} errors were found'.
format(errrCount)))
except:
tools.userMessage(
'ERRR: Error report dialog failed to work! (error_manager.py)'
)
if warnCount == 1:
tools.userMessage("WARN: Also, one warning was found too")
elif warnCount > 1:
tools.userMessage("WARN: A total " + str(warnCount) +
" warnings were found too")
def main (self) :
basePath = os.environ.get('PTXPLUS_BASE')
settings = tools.getSettingsObject()
# Before we do anything we need to check to see if the
# the version is not the same.
try :
curVer = settings['System']['General']['systemVersion']
except :
curVer = 0
tools.userMessage("Errr: Could not determine the current system version. Hopefully this update will fix that problem. Setting current version to 0")
if settings['System']['systemVersion'] != curVer :
# First make a backup copy of our original .project.conf
bakSettingsProjectFile = tools.getProjectConfigFileName() + '~'
settingsProjectFile = tools.getProjectConfigFileName()
shutil.copy(settingsProjectFile, bakSettingsProjectFile)
oldSettings = ConfigObj(bakSettingsProjectFile,encoding='utf-8')
# Get the system defaul .conf file
systemProjectConfFile = basePath + "/resources/lib_sysFiles/" + tools.getProjectConfigFileName()
tempMasterConfFile = os.getcwd() + "/" + tools.getProjectConfigFileName()
if os.path.isfile(tempMasterConfFile) != True :
tools.userMessage("Errr: Could not update project, [" + tempMasterConfFile + "] not found")
else :
# Copy into the project folder
shutil.copy(systemProjectConfFile, tempMasterConfFile)
# Load in the object
masterSettings = ConfigObj(tempMasterConfFile,encoding='utf-8')
# Now we will do a simple merge and overwrite the users projet \
# settings over our copy of the new settings
masterSettings.merge(oldSettings)
# Set it to the right version
masterSettings['System']['General']['systemVersion'] = settings['System']['systemVersion']
# Now we will write out the results to our new master copy
masterSettings.write()
tools.userMessage("Info: Version numbers did not match, project was updated from: " + curVer + " to: " + masterSettings['System']['General']['systemVersion'])
return True
else :
# tools.userMessage("Info: Version numbers between the system and the project matched. Project was not updated.")
return False
def main (self, insertType, wikiFile) :
'''This manages the process of writing to project wikis'''
# Get the user name
userName = tools.getSystemUser()
# Add a time stamp for notes and issues. More functions
# will be added as I think of them. :-)
if insertType == "issue" :
stamp = "**" + str(datetime.now()).split('.')[0] + "**\nLogged By: **" + userName + "**\nStatus: **Open**\n<Enter Issue Here>\n\n"
if tools.prependText(stamp, wikiFile) != True :
# This doesn't support log file output so for
# now we'll just write a simple error to the
# terminal if it fails.
tools.userMessage('Error: write_to_wiki.py has failed to output to the target wiki file which is: [' + wikiFile + '] The insertType used was: [' + insertType + ']')
elif insertType == "note" :
stamp = "**" + str(datetime.now()).split('.')[0] + "**\nLogged By: **" + userName + "**\n<Enter Note Text>\n\n"
if tools.prependText(stamp, wikiFile) != True :
# This doesn't support log file output so for
# now we'll just write a simple error to the
# terminal if it fails.
tools.userMessage('Error: write_to_wiki.py has failed to output to the target wiki file which is: [' + wikiFile + '] The insertType used was: [' + insertType + ']')
elif insertType == "about" :
print "Writing about file."
settings = tools.getSystemSettings()
aboutText = settings['System']['aboutText']
object = codecs.open(wikiFile, "w", encoding='utf_8')
object.write('=About: ptxplus=\n')
object.write('Version ' + '\n\n')
object.write(aboutText)
object.close()
else:
tools.userMessage('Error: write_to_wiki.py did not complete the process. It does not support the [' + insertType + '] insert type. This is an incorrect value passed by makefile.')
def main (self, log_manager) :
settings = tools.getSettingsObject()
hyphenPath = tools.pubInfoObject['Paths']['PATH_HYPHENATION']
# Set the output file name and the wordlist file name
texHyphenFileName = hyphenPath + '/' + tools.pubInfoObject['Files']['FILE_HYPHENATION_TEX']
wordListFileName = hyphenPath + '/' + tools.pubInfoObject['Files']['FILE_HYPHENATION']
lcCodeListFileName = hyphenPath + '/' + tools.pubInfoObject['Files']['FILE_LCCODELIST']
# Get our project hyphenation commands
languageCode = settings['ProjectText']['languageCode']
setHyphenCharacter = settings['Format']['Hyphenation']['setHyphenCharacter']
setHyphenPenalty = settings['Format']['Hyphenation']['setHyphenPenalty']
setExHyphenPenalty = settings['Format']['Hyphenation']['setExHyphenPenalty']
setPretolerance = settings['Format']['Hyphenation']['setPretolerance']
# If we see that the texHyphenFile exists we will check to see if
# the overwrite flag has been set.
if os.path.isfile(texHyphenFileName) == True and log_manager._optionalPassedVariable != 'overwrite' :
# Report that we found a .tex file and had to stop
tools.userMessage("WARN: " + texHyphenFileName + " exists. Process halted")
else :
# Just make the file, nothing else
# Open our wordlist file, if one exists, if not, make one
if not os.path.isfile(wordListFileName) :
word_list_in = codecs.open(wordListFileName, mode='w', encoding='utf_8')
else :
# Use utf_8_sig to open it in case it has a BOM in it!
word_list_in = tools.normalize(codecs.open(wordListFileName, mode='r', encoding='utf_8_sig'))
# Make the TeX hyphen file
tex_hypens_out = codecs.open(texHyphenFileName, "w", encoding='utf_8')
# Make header line
tex_hypens_out.write(
"% hyphenation.tex\n"
"% This is an auto-generated hyphenation rules file for this project.\n"
"% Please refer to the documentation for details on how to make changes.\n\n")
# Insert the TeX hyphenation commands from our .project.conf file
tex_hypens_out.write('\\newlanguage\\' + languageCode + 'language\n')
tex_hypens_out.write('\\language = \\' + languageCode + 'language\n')
tex_hypens_out.write('\\defaulthyphenchar=' + setHyphenCharacter + '\n')
tex_hypens_out.write('\\hyphenpenalty=' + setHyphenPenalty + '\n')
tex_hypens_out.write('\\exhyphenpenalty=' + setExHyphenPenalty + '\n')
if setPretolerance != '' :
tex_hypens_out.write('\\pretolerance=' + setPretolerance + '\n')
# Spacer
tex_hypens_out.write('\n\n')
# It may be necessary to have an lcCodeList included. These codes are
# kept in an external file normally kept in the project hyphenation folder.
if os.path.isfile(lcCodeListFileName):
tex_hypens_out.writelines(tools.normalize(codecs.open(lcCodeListFileName, 'r', encoding='utf_8')))
tex_hypens_out.write('\n')
# The hyphenation word list is normally generated in another process
# or it could be made by hand. It is normally kept in the project
# hyphenation folder. This next block of code will copy across the
# contents of the wordlist, skipping comments as we go.
tex_hypens_out.write('\hyphenation{\n')
tex_hypens_out.writelines(l for l in (l.lstrip() for l in word_list_in) if l[0] is not '%')
tex_hypens_out.write('}\n')
tex_hypens_out.close()
# Tell the world what we did
tools.userMessage("INFO: Created: " + texHyphenFileName)
basePath = os.environ.get('PTXPLUS_BASE')
if not basePath:
basePath = "/usr/share/xetex-ptxplus"
os.environ['PTXPLUS_BASE'] = basePath
sys.path.append(basePath + '/bin/python')
sys.path.append(basePath + '/bin/python/lib_system')
sys.path.append(basePath + '/bin/python/lib_scripture')
# First position in the command line arg. is the task.
# We have to have that or the process fails
try:
task = sys.argv[1]
except:
tools.userMessage(
"process_text.py: Cannot run the process because no module (task) has been specified."
)
sys.exit(1)
# Second position we add the file ID here so we can
# track what we are working on if we need to.
try:
typeID = sys.argv[2]
except:
typeID = "NA"
# In the third arg we have the input file name.
# There may be cases where this is not needed but
# this position always refers to the input file.
try:
inputFile = sys.argv[3]
def main(self):
# This needs to be rewritten to use the sfm parser which would simplify it somewhat
# Pull in any argments we need. For this module we know that the first one
# is the one we need. If there are more they are irrelevant.
adjustParaLength = int(0)
adjustParaLength = int(tools.getModuleArguments()[0])
# How do we do integer comparison?
if adjustParaLength <= 0:
adjustParaLength = int(29)
self._log_manager.log(
"ERRR",
"The paragraph length does not seem to be set. A default value will be used. Please check your settings file."
)
verseNumberMarker = "\\" + tools.pubInfoObject['USFM'][
'ChaptersVerses']['verseNumber']
footnoteOpenMarker = "\\" + tools.pubInfoObject['USFM']['Footnotes'][
'footnoteOpenMarker']
if os.path.isfile(self._outputFile):
# If it exists that may be a problem as we don't want to
# accidently wipe out any adjustment data that has been
# entered by the user. For this reason. we will not complete
# the process if we find one. We'll just output a warning
# to the log and exit gracefully.
self._log_manager.log(
"INFO", "The file " + self._outputFile +
" already exists so I will not build a new one.")
# Don't think we need to report this in the terminal
#head, tail = os.path.split(self._outputFile)
#tools.userMessage("ERRR: File found: " + tail)
return
# Get our book object - Using utf_8_sig because the source
# might be coming from outside the system and we may need
# to be able to handle a BOM.
try:
inputObject = codecs.open(self._inputFile,
"r",
encoding='utf_8_sig')
# Continue on by opening up a new .adj file
outputObject = codecs.open(self._outputFile, "w", encoding='utf_8')
except:
head, tail = os.path.split(self._inputFile)
tools.userMessage("ERRR: Parent not found: " + tail)
self._log_manager.log(
"ERRR", "The file: " + self._inputFile +
" could not be opened. Process halted")
return
self._log_manager.log(
"INFO", "Identifying all paragraphs with " +
str(adjustParaLength) + " words or more.")
paragraph = "off"
verseCount = 0
footnoteCount = 0
wordCount = 0
paragraphType = ""
locationLine = ""
for line in inputObject:
# First let's keep track of where we are in the book
self._markup_manager.setBookChapterVerse(line)
# Look for paragraph elements
wordsInLine = line.split()
# Count the words in this line
if paragraph == "on":
if len(wordsInLine) > 0:
# Look to see if this is a verse line
if wordsInLine[0] == verseNumberMarker:
if verseCount == 0:
# The output needs a special format so we need to get each component.
self._bookID = self._markup_manager.getBookID()
chapter = str(
self._markup_manager.getChapterNumber())
verse = str(self._markup_manager.getVerseNumber())
locationLine = "%" + self._bookID + " " + chapter + "." + verse + " +1"
if self._markup_manager._footnote_tracker.hasFootnoteOpenMarkerInLine(
line) == True:
footnoteCount += 1
verseCount += 1
else:
verseCount += 1
wordCount += len(wordsInLine)
else:
# Need to do some checking on other elements in
# the line. If a footnote marker is found we
# will go on to the next line. Otherwise we'll
# ship this one out. (Don't want to break up \f)
if line.find(footnoteOpenMarker) < 0:
self.writeAdjLine(verseCount, footnoteCount, \
wordCount, locationLine, paragraphType, \
adjustParaLength, outputObject)
# Paragraph gets turned off now. We are only
# looking for paragraph-type text containers.
paragraph = "off"
verseCount = 0
footnoteCount = 0
wordCount = 0
paragraphType = ""
locationLine = ""
# We'll just go one level deep on poetry (\q should be \q1)
if len(wordsInLine) > 0:
if wordsInLine[0] == "\\" + self._paragraphMarkers['paragraphNormal'] or \
wordsInLine[0] == "\\" + self._paragraphMarkers['paragraphLeft'] or \
wordsInLine[0] == "\\" + self._poetryMarkers['poeticLineOne'] :
paragraph = "on"
if self._markup_manager._footnote_tracker.hasFootnoteOpenMarkerInLine(
line) == True:
footnoteCount += 1
wordCount += len(wordsInLine)
if wordsInLine[0] == "\\" + self._paragraphMarkers[
'paragraphNormal']:
paragraphType = self._paragraphMarkers[
'paragraphNormal']
elif wordsInLine[0] == "\\" + self._paragraphMarkers[
'paragraphLeft']:
paragraphType = self._paragraphMarkers['paragraphLeft']
elif wordsInLine[
0] == "\\" + self._poetryMarkers['poeticLineOne']:
paragraphType = self._poetryMarkers['poeticLineOne']
else:
paragraphType = "##"
continue
# Clean up! Write out the last line now that we are done.
# As before, we need to do some checking on other elements in
# the line. If a footnote marker is found we will go on to
# the next line. Otherwise we'll ship this one out. (Don't
# want to break up a \f)
if line.find(footnoteOpenMarker) < 0:
self.writeAdjLine(verseCount, footnoteCount, wordCount, \
locationLine, paragraphType, adjustParaLength, outputObject)
# All done, tell the world what we did
head, tail = os.path.split(self._outputFile)
tools.userMessage("INFO: Created: " + tail)
self._log_manager.log(
"INFO", "Created paragraph adjustment file: " + self._outputFile)
self._log_manager.log(
"INFO", "Lines written = " + str(self._adjustLinesWritten))
def main(self):
'''We will open up our captions file which should be Unicode
encoded and in CSV format. The illustration IDs will
be matched from that file with the lib data file and
will create a piclist file for the book that is
currently being processed.'''
# Before we start we need to be sure our init succeeded so
# we will run some tests here.
# See if the output file already exists. if it does, then we stop here
if os.path.isfile(self._outputFile):
head, tail = os.path.split(self._outputFile)
tools.userMessage("INFO: " + tail + " exists")
self._log_manager.log(
"INFO", "The " + self._outputFile +
" exists so the process is being halted to prevent data loss.")
self._errors += 1
# Check to see if the captions file exists in the share folder
# if it doesn't we're all done for now
if not os.path.isfile(self._projectIllustrationsCaptions):
self._log_manager.log(
"ERRR", "The illustration caption file (" +
self._projectIllustrationsCaptions +
") is missing from the project. This process cannot work without it."
)
self._errors += 1
# If a path is defined, check to see if the path to the illustrations
# lib is good. If there is no path then we assume a custom lib is being
# used and that is handled different.
if self._sourceIllustrationsLibPath != '':
if not os.path.isdir(self._sourceIllustrationsLibPath):
self._log_manager.log(
"ERRR", "The path to the illustrations library (" +
self._sourceIllustrationsLibPath +
") does not seem to be correct. This process cannot work without it."
)
self._errors += 1
# If a data file is defined, check to see if the data file exists. If
# it doesn't we're done because we need that too. Again, if there is no
# file assigned, we are assuming a custom set of images.
if self._sourceIllustrationsLibData != '':
if not os.path.isfile(self._sourceIllustrationsLibData):
self._log_manager.log(
"ERRR", "The illustration data file (" +
self._sourceIllustrationsLibDataFileName +
") seems to be missing from the library. This process cannot work without it."
)
self._errors += 1
# If we get an error we really can't go on at this point
if self._errors != 0:
return
# If we are not in custom lib mode, pull in the library data file using
# the CSVtoDict class in tools
if self._sourceIllustrationsLibData != '':
try:
self._libData = tools.CSVtoDict(
self._sourceIllustrationsLibData)
except:
self._log_manager.log(
"ERRR",
"Not able to find (" + self._sourceIllustrationsLibData +
"). More than likely the file is missing or the path is wrong."
)
else:
# If there is not lib, then this file needs to be empty.
self._libData = ''
# If we didn't bail out right above, we'll go ahead and open the data file
# The assumption here is that the encoding of the pieces of the csv are
# what they need to be.
# Filter out any IDs that do not have anything to do with this book
# This fails when there is a blank line at the end of the CSV file.
# We'll put a try statement to catch it.
hits = ""
inFileData = ""
try:
inFileData = filter(
lambda l: l[1].lower() == self._bookID.lower(),
csv.reader(open(self._projectIllustrationsCaptions),
dialect=csv.excel))
# Right here we will sort the list by BCV. This should prevent unsorted
# data from getting out into the piclist.
inFileData.sort(cmp=lambda x, y: cmp(x[1], y[1]) or cmp(
int(x[2]), int(y[2])) or cmp(int(x[3]), int(y[3])))
# Do not process unless we are in the right book and
# keep track of the hits for this book
hits = 0
for line in inFileData:
if self._bookID.upper() == line[1].upper():
hits += 1
# If this next process fails, should we stop here? Hmmm...
self.processIllustrationFile(line[0])
except:
self._log_manager.log(
"ERRR",
"Failed to process all the lines in the source illustrations file. (Check for a blank on the end of the file.)"
)
# Now we need output anything we might have collected. If nothing was
# found, we will just send a simple message to the terminal to tell the
# user what happened
if hits > 0:
self._outFileObject = codecs.open(self._outputFile,
"w",
encoding='utf_8')
self._log_manager.log("DBUG", "Created file: " + self._outputFile)
tools.userMessage("INFO: Created piclist file for: " +
self._bookID.upper() + " (" + str(hits) + ")")
self._outFileObject.writelines(
self.collectPicLine(*line) + '\n' for line in inFileData)
# Close the piclist file
self._outFileObject.close()
else:
tools.userMessage("INFO: No illustrations found for: " +
self._bookID.upper())
# Tell the world what we did
self._log_manager.log(
"INFO", "We processed " + str(hits) +
" illustration line(s) for: " + self._bookID)
def main (self, log_manager) :
'''This is the main process function for getting and checking
project assets.'''
tools.userMessage('INFO: Checking project assets')
# Set the mode
self._log_manager = log_manager
self._mode = self._log_manager._optionalPassedVariable
if self._mode == '' :
self._mode = 'basic'
# Gather up the initial settings
basePath = os.environ.get('PTXPLUS_BASE')
baseSysLib = basePath + '/resources/lib_sysFiles'
pathHome = os.path.abspath(tools.pubInfoObject['Paths']['PATH_HOME'])
pathAdmin = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_ADMIN']
pathWiki = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_WIKI']
pathFonts = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_FONTS']
pathHyphenation = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_HYPHENATION']
pathTexts = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_TEXTS']
pathDeliverables = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_DELIVERABLES']
pathProcess = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_PROCESS']
pathMaps = pathHome + '/' + tools.pubInfoObject['Paths']['PATH_MAPS']
pathSource = os.path.abspath(self._log_manager._settings['System']['Paths'].get('PATH_SOURCE', '../Source'))
pathIllustrations = os.path.abspath(self._log_manager._settings['System']['Paths'].get('PATH_ILLUSTRATIONS', '../Source/Illustrations'))
pathUserLibFonts = os.path.abspath(self._log_manager._settings['System']['Paths']['PATH_FONT_LIB'])
pathUserLibGraphics = os.path.abspath(self._log_manager._settings['System']['Paths']['PATH_GRAPHICS_LIB'])
# This can be optional if a custom illustration lib is used
if self._log_manager._settings['System']['Paths']['PATH_ILLUSTRATIONS_LIB'] != '' :
pathUserLibIllustrations = os.path.abspath(self._log_manager._settings['System']['Paths']['PATH_ILLUSTRATIONS_LIB'])
else :
pathUserLibIllustrations = ''
pathIllustrationsLib = tools.pubInfoObject['Paths']['PATH_RESOURCES_ILLUSTRATIONS'].replace('__PTXPLUS__', basePath)
fileWatermark = self._log_manager._settings['Format']['PageLayout']['FILE_WATERMARK']
filePageBorder = self._log_manager._settings['Format']['PageLayout']['FILE_PAGE_BORDER']
listGraphics = self._log_manager._settings['Format']['Illustrations']['LIST_GRAPHICS']
pathPeripheral = pathSource + '/' + os.getcwd().split('/')[-1]
# Do some sanity testing
if not os.path.isdir(pathUserLibFonts) :
self._log_manager.log('WARN', 'No user font library folder found. Please check your configuration.', 'true')
if not os.path.isdir(pathUserLibGraphics) :
self._log_manager.log('WARN', 'No user graphics library folder found. Please check your configuration.', 'true')
# Don't bother doing the test if this is set to null
if pathUserLibIllustrations != '' :
if not os.path.isdir(pathUserLibIllustrations) :
self._log_manager.log('WARN', 'No user Illustrations library folder found. Please check your configuration.', 'true')
# Check/install folders we might need
if not os.path.isdir(pathSource) :
os.mkdir(pathSource)
self._log_manager.log('INFO', 'Added Source folder', 'true')
# Make the peripheral folder inside Source
if not os.path.isdir(pathPeripheral) :
os.mkdir(pathPeripheral)
self._log_manager.log('INFO', 'Added Peripheral matter folder (in Source):', 'true')
# Make the Process folder, we will always need that
if not os.path.isdir(pathProcess) :
os.mkdir(pathProcess)
self._log_manager.log('INFO', 'Added Process folder', 'true')
tools.copyAll(baseSysLib + '/Process', pathProcess)
self._log_manager.log('INFO', 'Copied new process files to project', 'true')
# If there are no map components then there is no need to make the folder
if len(self._log_manager._settings['Format']['BindingGroups']['GROUP_MAPS']) > 0 :
if not os.path.isdir(pathMaps) :
os.mkdir(pathMaps)
self._log_manager.log('INFO', 'Added Maps folder', 'true')
# Make the illustrations folder inside Source
if not os.path.isdir(pathIllustrations) :
os.mkdir(pathIllustrations)
self._log_manager.log('INFO', 'Added shared Illustrations folder (in Source)', 'true')
# If it is turned on, make the hyphenation folder
# and populate it with the necessary files
if self._log_manager._settings['Format']['Hyphenation']['useHyphenation'].lower() == 'true' :
if not os.path.isdir(pathHyphenation) :
os.mkdir(pathHyphenation)
self._log_manager.log('INFO', 'Added Hyphenation folder', 'true')
tools.copyAll(baseSysLib + '/Hyphenation', pathHyphenation)
self._log_manager.log('INFO', 'Copied hypheation files to project', 'true')
# Create the project wiki folder and populate
# it with the necessary files
if not os.path.isdir(pathWiki) :
os.mkdir(pathWiki)
self._log_manager.log('INFO', 'Added .wiki folder (hidden)', 'true')
tools.copyAll(baseSysLib + '/Wiki', pathWiki)
self._log_manager.log('INFO', 'Copied fresh wiki files to project', 'true')
# Make the Process folder, we will always need that
if not os.path.isdir(pathDeliverables) :
os.mkdir(pathDeliverables)
self._log_manager.log('INFO', 'Added Deliverables folder', 'true')
# Make the Texts folder, we will always need that too
if not os.path.isdir(pathTexts) :
os.mkdir(pathTexts)
self._log_manager.log('INFO', 'Added Texts folder', 'true')
# Make the admin folder if an admin code has been given
# This should be a one-time event
# NOTE: The act of copying in all the possible admin forms, etc is being
# depricated. This tends to not be useful in this context. We will create
# the Admin folder but nothing else.
if not os.path.isdir(pathAdmin) :
os.mkdir(pathAdmin)
self._log_manager.log('INFO', 'Added Admin folder', 'true')
# eCode = self._log_manager._settings['Project'].get('entityCode', '').lower()
# if eCode != '' :
# if not os.path.isdir(pathAdmin) :
# os.mkdir(pathAdmin)
# self._log_manager.log('INFO', 'Added Admin folder', 'true')
# # Now copy the files in that are for this entity
# tools.copyAll(baseSysLib + '/Admin/' + eCode, pathAdmin)
# self._log_manager.log('INFO', 'Copied entity admin files to project', 'true')
# The font folder will be a little more complex
# If no fonts are listed or the setting is missing for some
# reason we will run with the default of CharisSIL.
sysFontFolder = self.subBasePath(tools.pubInfoObject['Paths']['PATH_RESOURCES_FONTS'], basePath)
fontList = self._log_manager._settings['Format']['Fonts'].get('fontFamilyList', 'CharisSIL')
if not os.path.isdir(pathFonts) :
os.mkdir(pathFonts)
self._log_manager.log('INFO', 'Added Fonts folder', 'true')
# We will not copy any files from the source folder now.
# At this point the only file there should be the font
# config file. That is copied in when the localizing is
# done to the fonts in the project (below).
# We assume that the font which is in the users resource
# lib is best so we will look there first for our fonts.
# If we don't find it there we will try to get it from
# the system font folder. We will report any that we
# don't find.
for ff in fontList :
os.mkdir(pathFonts + '/' + ff)
# First check our resource font folder
if os.path.isdir(pathUserLibFonts + '/' + ff) :
tools.copyFiles(pathUserLibFonts + '/' + ff, pathFonts + '/' + ff)
self._log_manager.log('INFO', 'Copied [' + ff + '] font family', 'true')
# If not there, then get what you can from the system font folder
else :
if os.path.isdir(sysFontFolder + '/' + ff) :
tools.copyFiles(sysFontFolder + '/' + ff, pathFonts + '/' + ff)
self._log_manager.log('INFO', 'Copied [' + ff + '] font family', 'true')
else :
self._log_manager.log('ERRR', 'Not able to copy [' + ff + '] font family', 'true')
# FIXME:
# We don't need to localize the fonts every time but if there is a change
# to one of the font settings it would be good to do that. Also, the first
# time the project is run on another system it needs to be done too.
# However, we do not have a good way yet to determine either so we have to
# localize every time this is run, unfortunately.
self.localiseFontsConf(pathFonts, sysFontFolder)
# Check/install system assets
# Watermark
self.smartCopy(pathUserLibGraphics + '/' + fileWatermark, pathIllustrations + '/' + fileWatermark, pathProcess + '/' + fileWatermark, pathIllustrationsLib + '/' + fileWatermark)
# Page border
self.smartCopy(pathUserLibGraphics + '/' + filePageBorder, pathIllustrations + '/' + filePageBorder, pathProcess + '/' + filePageBorder, pathIllustrationsLib + '/' + filePageBorder)
# Graphics list
for graphic in listGraphics :
self.smartCopy(pathUserLibGraphics + '/' + graphic, pathIllustrations + '/' + graphic, pathProcess + '/' + graphic, pathIllustrationsLib + '/' + graphic)
def deleteErrorLogs(self):
'''Get rid of the error log files.'''
if os.path.isfile(self._errorLogFile) == True:
if os.system("rm " + self._errorLogFile) != 0:
tools.userMessage("Failed to delete: " + self._errorLogFile)
def main(self, filename):
'''This is the process function for generating a project archive.
We assume that we are starting in the project folder.'''
# Get the project ID and path
projectID = tools.getProjectID()
projectPath = os.getcwd()
self.settings = tools.getSettingsObject()
print "File: " + filename
# Output an archive.conf file that contains all the current settings
# Though this is just a one-time operation it is easier to do it in tools
archiveConfObject = tools.getProjectSettingsObject()
if archiveConfObject != None:
archiveConfObject['Process']['General']['LOCKED'] = 1
archiveConfObject.filename = 'archive.conf'
archiveConfObject.write()
# Move into the parent folder to start the process (We want to
# get the source files as well as the project files
projHome, proj = os.path.split(projectPath)
os.chdir(projHome)
# Build a path to the archive folder default is ../../Archive
# If it is not the default we'll assume the path is absolute
if filename == projectID:
archivePath = self.settings['General']['Archive']['archivePath']
if archivePath.find('../..') > -1:
archivePath = os.getcwd() + '/Archive'
archiveFile = archivePath + "/" + projectID + ".tar.gz"
# Look and see if the Archive folder exists
if not os.path.isdir(archivePath):
os.mkdir(archivePath)
else:
archiveFile = filename
# Look to see if the archiveFile exists, if so, we'll delete it
if os.path.isfile(archiveFile):
os.remove(archiveFile)
# Now we'll move back into the project folder.
os.chdir(projectPath)
# Make the file list we want to tar.
fileList = []
for root, dirs, files in tools.walk('.'):
# print root, dirs
# Filter out all the files we don't want and
# include a few we want to keep.
for name in files:
self.pruneDirs(root, dirs)
if self.excludeFileFilter(name) != True and \
self.excludeFileTypeFilter(name) != True or \
self.includeFileFilter(name) == True :
fileList.append(os.path.join(root, name))
if root == '.':
self.pruneDirs(root, dirs)
# Now make the archive file. However, we want to keep it from adding
# any backup files and the .svn folders. This should make for eaiser
# transfer between systems.
tar = tarfile.open(archiveFile, 'w:gz')
for file in fileList:
tar.add(file)
tar.close()
# Tell the world what we did
tools.userMessage('Archived project: ' + projectID)
def main(self, log_manager):
'''This is the main process function for regression testing.'''
# INITIAL SETTINGS
self._log_manager = log_manager
useRegressionTests = self._log_manager._settings['System']['General'][
'useRegressionTests']
visualDiffChecking = self._log_manager._settings['System']['General'][
'visualDiffChecking']
basePath = os.environ.get('PTXPLUS_BASE')
pathHome = os.path.abspath(tools.pubInfoObject['Paths']['PATH_HOME'])
pathRegression = os.path.abspath(
tools.pubInfoObject['Paths']['PATH_REGRESSION'])
new = self._log_manager._currentInput
old = ''
newFile = ''
oldFile = ''
switch = self._log_manager._optionalPassedVariable
# SET TYPE (BUILD ADDITIONAL PATHS)
self.buildFolderPath(pathRegression)
# Set the test type (if we don't know, we don't go)
if os.path.exists(new):
if os.path.isfile(new):
testType = 'file'
# Do some file name manipulation to get our old and new file names
# It is assumed that we are working in the "Texts" folder and no
# other. This might come back to bite us at some point.
head, tail = os.path.split(new)
old = pathRegression + '/' + head.split('/')[-1]
# Build the folder path if needed so processes below will not fail
self.buildFolderPath(old)
newFile = new
oldFile = old + '/' + tail
self._log_manager.log(
'INFO',
'Regression type is file (' + os.path.split(new)[-1] + ')',
'true')
elif os.path.isdir(new):
testType = 'dir'
head, tail = os.path.split(new)
old = pathRegression + '/' + tail
self._log_manager.log(
'INFO', 'Regression test type is folder (' +
str(len(os.listdir(new))) + ' items)', 'true')
else:
self._log_manager.log(
'ERRR',
'Regression testing for [' + new + '] target type unknown',
'true')
sys.exit(1)
else:
self._log_manager.log(
'WARN', 'Regression test, target not found: [' + new + ']',
'true')
sys.exit(1)
# MASTER SWITCH
# Do not do anything if useRegressionTests is set to false
if useRegressionTests.lower() == 'false':
self._log_manager.log('INFO',
'Regression tests have been turned off',
'true')
return
else:
tools.userMessage('INFO: Starting regression tests')
# UTILITY FUNCTIONS
# There are some small tasks that can be done by this module
# and are triggered by an optionally passed switch var.
# Set the current component as the regression base file
if switch.lower() == 'set':
# Deal with any exsiting file
if os.path.isfile(oldFile):
os.unlink(oldFile)
# Copy file and set permission to read-only
shutil.copy(newFile, oldFile)
os.chmod(oldFile, stat.S_IREAD)
self._log_manager.log(
'INFO', 'Setting the current component as base (' +
os.path.split(new)[-1] + ')', 'true')
# SANITY TESTING
# If we survived to this point, do some sanity testing
# First see if the regression testing folder is there
# then copy whatever files are needed.
if not os.path.isdir(old):
self._log_manager.log('INFO',
'No regression folder found, creating now',
'true')
os.makedirs(old)
if testType == "file":
# Copy one file and set to readonly
shutil.copy(newFile, oldFile)
os.chmod(oldFile, stat.S_IREAD)
elif testType == "dir":
# Copy all the files and set all of them to readonly
tools.copyFiles(new, old)
tools.chmodFiles(old, stat.S_IREAD)
# Other tests for whole folder
if os.path.isdir(new):
# If there are absolutely no files in the regression folder for
# for some odd reason, copy them over from the new target
# and make them all readonly
if len(os.listdir(old)) == 0:
self._log_manager.log(
'INFO',
'No items in regression folder, copying from new target',
'true')
if testType == "file":
# Copy one file and set to readonly
os.unlink(oldFile)
shutil.copy(newFile, oldFile)
os.chmod(oldFile, stat.S_IREAD)
elif testType == "dir":
# Copy all the files and set all of them to readonly
tools.unlinkFiles(old)
tools.copyFiles(new, old)
tools.chmodFiles(old, stat.S_IREAD)
# If the number of files is unequal, alert the user and
# let them sort it out. It would be nice if this was more
# automated but it is better to be manual at this point to
# be safe.
if len(os.listdir(new)) != len(os.listdir(old)):
self._log_manager.log(
'WARN', 'Base folder out of sync with new target', 'true')
self._log_manager.log(
'INFO', 'Target = ' + str(len(os.listdir(new))) +
' - Base = ' + str(len(os.listdir(old))), 'true')
# Other tests for individual component
elif os.path.isfile(newFile):
if not os.path.isfile(oldFile):
self._log_manager.log('INFO',
'No base file found, creating now',
'true')
shutil.copy(newFile, oldFile)
os.chmod(oldFile, stat.S_IREAD)
# DIFF CHECK
# Folder Diff test
if testType.lower() == 'dir':
# Build the command
sysCommand = "diff " + old + " " + new + " > /dev/null"
self._log_manager.log('INFO', 'Doing initial folder DIFF check',
'true')
# Send off the command return error code
if os.system(sysCommand) != 0:
self.openMeld(old, new, visualDiffChecking)
else:
self._log_manager.log(
'INFO',
'No problems found, regression test check complete',
'true')
# File Diff test
elif testType.lower() == 'file':
# Build the command
sysCommand = "diff " + oldFile + " " + newFile + " > /dev/null"
self._log_manager.log('INFO', 'Doing initial file DIFF check',
'true')
# Send off the command return error code
if os.system(sysCommand) != 0:
self.openMeld(oldFile, newFile, visualDiffChecking)
else:
self._log_manager.log(
'INFO',
'No problems found, regression test check complete',
'true')
