Example #1
0
class ProjData (object) :

    def __init__(self, pid, gid = None) :
        '''Intitate the whole class and create the object.'''

        self.pid            = pid
        self.gid            = gid
        self.tools          = Tools()
        self.user           = UserConfig()
        self.userConfig     = self.user.userConfig
        self.local          = ProjLocal(pid)
        self.log            = ProjLog(pid)
        self.projHome       = os.path.join(os.path.expanduser(self.userConfig['Resources']['projects']), self.pid)
        self.projList       = self.tools.getProjIdList(os.path.expanduser(self.userConfig['Resources']['projects']))

        # Log messages for this module
        self.errorCodes     = {

            '1220' : ['LOG', 'Project [<<1>>] already registered in the system.'],
            '1240' : ['ERR', 'Could not find/open the Project configuration file for [<<1>>]. Project could not be registered!'],

            '3410' : ['LOG', 'Backup file cull skipping: [<<1>>] Not a recognized Rapuma backup file name format.'],
            '3510' : ['ERR', 'The path (or name) given is not valid: [<<1>>].'],
            '3530' : ['MSG', 'Project backup: [<<1>>] has been restored to: [<<2>>]. A backup of the orginal project remains and must be manually removed.'],
            '3550' : ['ERR', 'Project backup version request: [<<1>>] exceeds the maxium number which could be in storage which is: [<<2>>]. Request an earlier (lesser) version.'],
            '3610' : ['ERR', 'The [<<1>>]. project is not registered. No backup was done.'],
            '3620' : ['ERR', 'The path to the backup folder is not valid [<<1>>]. Please try again.'],
            '3622' : ['ERR', 'The path to the backup folder is not set. Please set it and try again.'],
            '3625' : ['ERR', 'The path given to the backup folder is not valid [<<1>>]. Please set the system backup path.'],
            '3630' : ['MSG', 'Backup for [<<1>>] created and saved to: [<<2>>]'],

            '4110' : ['MSG', 'Completed merging data.'],
            '4120' : ['MSG', 'No files updated.'],
            '4130' : ['MSG', 'Added: <<1>> file(s).'],
            '4140' : ['MSG', 'Updated: <<1>> file(s)'],
            '4150' : ['WRN', 'The project data in: [<<1>>] will be replaced with the data from: [<<2>>].'],

            '4210' : ['MSG', 'Completed pulling/restoring data from the cloud.'],
            '4220' : ['ERR', 'Cannot resolve path: [<<1>>]'],
            '4250' : ['ERR', 'The cloud project [<<1>>] you want to pull from is owned by [<<2>>]. Use force (-f) to pull the project and change the local owner ID.'],
            '4260' : ['ERR', 'The local project [<<1>>] is newer than the cloud copy. If you seriously want to overwrite it, use force (-f) to do so.'],
            '4270' : ['MSG', 'Restored the project [<<1>>] from the cloud copy. Local copy is owned by [<<2>>].'],

        }


###############################################################################
############################## General Functions ##############################
###############################################################################
####################### Error Code Block Series = 1000 ########################
###############################################################################



###############################################################################
########################## Archive Project Functions ##########################
###############################################################################
####################### Error Code Block Series = 2000 ########################
###############################################################################

    def makeExcludeFileList (self, source) :
        '''Return a list of files that are not necessary to be included in a backup
        template or an archive. These will be all auto-generated files that containe system-
        specific paths, etc.'''

        excludeFiles        = []
        excludeTypes        = ['delayed', 'log', 'notepages', 'parlocs', 'pdf', 'tex', 'piclist', 'adj', 'zip']
        excludeFolders      = ['Draft', 'Final', 'HelperScript', 'Proof']

        # Process the excluded folders
        for root, dirs, files in os.walk(source) :
            for fileName in files :
                if os.path.basename(root) in excludeFolders :
                    excludeFiles.append(os.path.join(root, fileName))
                else :
                    # Get rid of edited backup files
                    if fileName[-1] == '~' :
                        excludeFiles.append(os.path.join(root, fileName))
                        continue
                    ext = os.path.splitext(fileName)[1][1:]
                    if ext in excludeTypes :
                        # A special indicator for file we want to keep
                        if fileName.find('-ext.') > 0 :
                            continue

        return excludeFiles

# FIXME: Should archiveProject() use self.pid instead of explicitly passing in a pid?
    def archiveProject (self, pid, path = None) :
        '''Archive a project. Send the compressed archive file to the user-specified
        archive folder. If none is specified, put the archive in cwd. If a valid
        path is specified, send it to that location. Like backup, this too will
        overwrite any existing file of the same name. The difference is that this
        will also disable the project so it cannot be accesses by Rapuma. When a
        project is archived, all work should cease on the project.'''

        # Make a private project object just for archiving
        aProject = Project(pid, self.gid)
        # Set some paths and file names
        archName = aProject.projectIDCode + '.rapuma'
        userArchives = self.userConfig['Resources']['archive']
        archTarget = ''
        if path :
            path = self.tools.resolvePath(path)
            if os.path.isdir(path) :
                archTarget = os.path.join(path, archName)
            else :
                self.tools.terminal('\nError: The path given is not valid: [' + path + ']\n')
                self.tools.dieNow()
        elif os.path.isdir(userArchives) :
            archTarget = os.path.join(userArchives, archName)
        elif os.path.isdir(os.path.dirname(aProject.local.projHome)) :
            # Default to the dir just above the project
            archTarget = os.path.dirname(aProject.local.projHome)
        else :
            self.tools.terminal('\nError: Cannot resolve a path to create the archive file!\n')
            self.tools.dieNow()

        # Get a list of files we don't want
        excludeFiles = self.makeExcludeFileList(source)

        self.zipUpProject(archTarget, excludeFiles)

        # Rename the source dir to indicate it was archived
        bakArchProjDir = aProject.local.projHome + '(archived)'
        if os.path.isdir(bakArchProjDir) :
            self.tools.terminal('\nError: Cannot complete archival process!\n')
            self.tools.terminal('\nAnother archived version of this project exsits with the folder name of: ' + self.tools.fName(bakArchProjDir) + '\n')
            self.tools.terminal('\nPlease remove or rename it and then repete the process.\n')
            self.tools.dieNow()
        else :
            os.rename(aProject.local.projHome, bakArchProjDir)

        # Finish here
        self.tools.terminal('Archive for [' + pid + '] created and saved to: ' + archTarget + '\n')


    def zipUpProject (self, target, excludeFiles = None) :
        '''Zip up a project and deposit it to target location. Be sure to strip
        out all all auto-created, user-specific files that could mess up a
        transfer to another system. This goes for archives and backups'''

#        import pdb; pdb.set_trace()

        # In case an exclude list is not given
        if not excludeFiles :
            excludeFiles = []

        # Do the zip magic here
        root_len = len(self.local.projHome)
        with zipfile.ZipFile(target, 'w', compression=zipfile.ZIP_DEFLATED) as myzip :
            sys.stdout.write('Backing up files')
            sys.stdout.flush()
            for root, dirs, files in os.walk(self.local.projHome) :
                # Chop off the part of the path we do not need to store
                zip_root = os.path.abspath(root)[root_len:]
                for f in files :
                    if os.path.join(root, f) in excludeFiles :
                        continue
                    if not f[-1] == '~' :
                        fn, fx = os.path.splitext(f)
                        fullpath = os.path.join(root, f)
                        zip_name = os.path.join(zip_root, f)
                        sys.stdout.write('.')
                        sys.stdout.flush()
                        myzip.write(fullpath, zip_name, zipfile.ZIP_DEFLATED)
            # Add a space before the next message
            print '\n'

# FIXME: Should restoreArchive() use self.pid instead of explicitly passing in a pid?
    def restoreArchive (self, pid, targetPath, sourcePath = None) :
        '''Restore a project from the user specified storage area or sourcePath if 
        specified. Use targetPath to specify where the project will be restored.
        Rapuma will register the project there.'''

        # Check to see if the user included the extension
        try :
            pid.split('.')[1] == 'rapuma'
            archName = pid
            pid = pid.split('.')[0]
        except :
            archName = pid + '.rapuma'

        archSource = ''
        archTarget = ''
        userArchives = ''

        # First look for the archive that is to be restored
        if sourcePath :
            if os.path.isdir(sourcePath) :
                archSource = os.path.join(sourcePath, archName)
        elif os.path.isdir(self.userConfig['Resources']['archive']) :
            userArchives = self.userConfig['Resources']['archive']
            archSource = os.path.join(userArchives, archName)
        else :
            self.tools.terminal('\nError: The path (or name) given is not valid: [' + archSource + ']\n')
            self.tools.dieNow()

        # Now set the target params
        if targetPath :
            if not os.path.isdir(targetPath) :
                self.tools.terminal('\nError: The path given is not valid: [' + targetPath + ']\n')
                self.tools.dieNow()
            else :
                archTarget = os.path.join(targetPath, pid)

        # If we made it this far, extract the archive
        with zipfile.ZipFile(archSource, 'r') as myzip :
            myzip.extractall(archTarget)

        # Permission for executables is lost in the zip, fix it here
        for folder in ['Scripts', os.path.join('Macros', 'User')] :
            self.tools.fixExecutables(os.path.join(archTarget, folder))

# FIXME: This will need some work 

        # Add project to local Rapuma project registry
        # To do this we need to open up the restored project config file
        # and pull out some settings.
        local       = ProjLocal(pid)
        pc          = Config(pid)
        log         = ProjLog(pid)
        aProject    = Project(pid, self.gid)
    #    import pdb; pdb.set_trace()

        # Finish here
        self.tools.terminal('\nRapuma archive [' + pid + '] has been restored to: ' + archTarget + '\n')


###############################################################################
########################### Backup Project Functions ##########################
###############################################################################
####################### Error Code Block Series = 3000 ########################
###############################################################################

    def cullBackups (self, maxBak, bakDir) :
        '''Remove any excess backups from the backup folder in
        this project.'''

        # Get number of maximum backups to store
        maxStoreBackups = int(maxBak)
        if not maxStoreBackups or maxStoreBackups == 0 :
            maxStoreBackups = 1

        # Build the cullList
        cullList = []
        files = os.listdir(bakDir)
        for f in files :
            try :
                cullList.append(int(f.split('.')[0]))
            except :
                self.log.writeToLog(self.errorCodes['3410'], [f])
        # Remove oldest file(s)
        while len(cullList) > maxStoreBackups :
            fn = min(cullList)
            cullList.remove(min(cullList))
            os.remove(os.path.join(bakDir, str(fn) + '.zip'))


    def backupProject (self, targetPath=None) :
        '''Backup a project. Send the compressed backup file with a date-stamp
        file name to the user-specified backup folder. If a target path is 
        specified, put the archive there but use the PID in the name. If other
        backups with the same name exist there, increment with a number.'''

        # First see if this is even a valid project
        if self.pid not in self.projList :
            self.log.writeToLog(self.errorCodes['3610'], [self.pid])

        # Set some paths and file names
        if not targetPath :
            # Now check for a valid location to backup to
            if self.local.userLibBackup == '' :
                self.log.writeToLog(self.errorCodes['3622'])
            elif not os.path.exists(self.local.userLibBackup) :
                self.log.writeToLog(self.errorCodes['3620'], [self.local.userLibBackup])
            projBackupFolder    = os.path.join(self.local.userLibBackup, self.pid)
            backupTarget        = os.path.join(projBackupFolder, self.tools.fullFileTimeStamp() + '.zip')
        else :
            projBackupFolder    = self.tools.resolvePath(targetPath)
            # Now check for a valid target path
            if not os.path.exists(projBackupFolder) :
                self.log.writeToLog(self.errorCodes['3625'], [targetPath])
            backupTarget        = self.tools.incrementFileName(os.path.join(projBackupFolder, self.pid + '.zip'))

        # Make sure the dir is there
        if not os.path.exists(projBackupFolder) :
            os.makedirs(projBackupFolder)

#        import pdb; pdb.set_trace()

        # Zip up but use a list of files we don't want
        self.zipUpProject(backupTarget, self.makeExcludeFileList(source))

        # Cull out any excess backups
        if not targetPath :
            self.cullBackups(self.userConfig['System']['maxStoreBackups'], projBackupFolder)

        # Finish here
        pc = Config(self.pid)
        pc.getProjectConfig()
        pc.projectConfig['Backup']['lastBackup'] = self.tools.fullFileTimeStamp()
        self.tools.writeConfFile(pc.projectConfig)
        self.log.writeToLog(self.errorCodes['3630'], [self.pid,backupTarget])
        return True


    def backupRestore (self, backup, target = None) :
        '''Restore a backup to the current or specified project.'''

#        import pdb; pdb.set_trace()

        if not target :
            target = self.local.projHome

        # Now remove the orginal
        if os.path.exists(target) :
            shutil.rmtree(target)
        # Create an empty folder to restore to
        os.makedirs(target)

        # If we made it this far, extract the archive
        with zipfile.ZipFile(backup, 'r') as myzip :
            myzip.extractall(target)
        return True


#    def restoreLocalBackup (self, bakFile) :
#        '''Restore from a project backup. As a project may have multiple backups in
#        its backup folder, the user will need to provide a number from 1 to n (n being
#        the number of backups in the folder, 1 being the most recent and n being the
#        oldest). If no number is provided, 1, (the most recent) will be restored.'''

#        # Adjust bNum if needed
#        maxBak = int(self.userConfig['System']['maxStoreBackups'])
#        if not bNum :
#            bNum = 0
#        else :
#            bNum = int(bNum)
#            if bNum <= 0 :
#                bNum = 0
#            elif bNum > maxBak :
#                self.log.writeToLog(self.errorCodes['3550'], [str(bNum), str(maxBak)])
#            else :
#                bNum = bNum-1

#        # Get vals we need
#        projHome            = self.getProjHome()
#        projBackupFolder    = self.tools.resolvePath(os.path.join(self.userConfig['Resources']['backup'], self.pid))

#        # Get the archive file name
#        files = os.listdir(projBackupFolder)
#        fns = []
#        for f in files :
#            fns.append(int(f.split('.')[0]))
#        # Sort the list, last (latest) first
#        fns.sort(reverse=True)
#        # Make file path/name
#        backup = os.path.join(projBackupFolder, str(fns[bNum]) + '.zip')
#        if not os.path.exists(backup) :
#            self.log.writeToLog(self.errorCodes['3510'], [backup])

#        # Restore the backup
#        self.backupRestore(backup, projHome)

#        # Permission for executables is lost in the zip, fix them here
#        self.tools.fixExecutables(projHome)

#        # Add helper scripts if needed
#        if self.tools.str2bool(self.userConfig['System']['autoHelperScripts']) :
#            ProjCommander(self.pid).updateScripts()

#        # Finish here (We will leave the project backup in place)
#        self.log.writeToLog(self.errorCodes['3530'], [self.tools.fName(backup),projHome])


    def restoreExternalBackup (self, source, target = None, force = False) :
        '''Restore a non-existant project from an external backup to a target folder.
        If no target is provided the project will be installed in the default project
        folder. The source path and ZIP file must be valid'''

        # Get/make the (localized) project home reference
        projHome = self.getProjHome(target)

#        import pdb; pdb.set_trace()

        # Create the source backup file name
#        source = os.path.join(source, self.pid + '.zip')

# FIXME: This needs some review and rework

        # Restore the backup
        if self.backupRestore(source, projHome) :

            # Permission for executables is lost in the zip, fix them here
            self.tools.fixExecutables(projHome)

            # If this is a new project we will need to register it now
            self.registerProject(projHome)

            # Add helper scripts if needed
            if self.tools.str2bool(self.userConfig['System']['autoHelperScripts']) :
                ProjCommander(self.pid).updateScripts()

            # Finish here (We will leave the backup-backup in place)
            self.tools.terminal('\nRapuma backup [' + self.pid + '] has been restored to: ' + projHome + '\n')
            
            return True


###############################################################################
############################ Cloud Backup Functions ###########################
###############################################################################
####################### Error Code Block Series = 4000 ########################
###############################################################################


    def isNewerThanCloud (self, cloud, projectConfig) :
        '''Compare time stamps between the cloud and the local project.
        Return True if the local project is newer or the same age as
        the copy in the cloud. Return True if the project does not
        exist in the local copy of the cloud.'''

        # First see if it exists
        cConfig = self.getConfig(cloud)
        if not cConfig :
            return True
        elif not cConfig.has_key('Backup') :
            return True
        elif not cConfig['Backup'].has_key('lastCloudPush') :
            return True
        # Check local for key
        if not projectConfig.has_key('Backup') :
            return False
        elif not projectConfig['Backup'].has_key('lastCloudPush') :
            return False
        # Compare if we made it this far
        cStamp = cConfig['Backup']['lastCloudPush']
        lStamp = projectConfig['Backup']['lastCloudPush']
        if lStamp >= cStamp :
            return True


    def isNewerThanLocal (self, cloud, projectConfig) :
        '''Compare time stamps between the cloud and the local project.
        Return True if the cloud project is newer or the same age as
        the local copy. Return True if the project does not exist in
        as a local copy.'''

        # First see if the local exists
        if not projectConfig :
            return True

        # See if cloud is there and up-to-date
        cloudConfig = self.getConfig(cloud)
        if not cloudConfig :
            return False

        # Compare if we made it this far
        cStamp = cloudConfig['Backup']['lastCloudPush']
        # It is possible the local has never been pushed
        # If that is the case, local is assumed older
        try :
            pStamp = projectConfig['Backup']['lastCloudPush']
        except :
            return False
        if cStamp >= pStamp :
            return True


    def getConfig (self, projHome) :
        '''Return a valid config object from cloud project.'''

#        import pdb; pdb.set_trace()

        projectConfigFile = os.path.join(projHome, 'Config', 'project.conf')
        if os.path.exists(projectConfigFile) :
            return ConfigObj(projectConfigFile, encoding='utf-8')


    def getCloudOwner (self, cloud) :
        '''Return the owner of a specified cloud project.'''

        try :
            return self.getConfig(cloud)['Backup']['ownerID']
        except :
            return None


    def getLocalOwner (self) :
        '''Return the owner of a specified cloud project.'''

        return self.userConfig['System']['userID']


    def sameOwner (self, cloud) :
        '''Return True if the owner of a given cloud is the same as
        the system user. Also return True if the cloud owner is not
        present.'''

        # First check for existence
        if not self.getCloudOwner(cloud) :
            return True
        # Compare if we made it to this point
        if self.getCloudOwner(cloud) == self.getLocalOwner() :
            return True


    def setCloudPushTime (self, projectConfig) :
        '''Set/reset the lastPush time stamp setting.'''

        projectConfig['Backup']['lastCloudPush'] = self.tools.fullFileTimeStamp()
        self.tools.writeConfFile(projectConfig)


    def buyCloud (self, projectConfig) :
        '''Change the ownership on a project in the cloud by assigning
        your userID to the local project cloudOwnerID. Then, using force
        the next time the project is pushed to the cloud, you will own it.'''

        projOwnerID = self.userConfig['System']['userID']
        projectConfig['Backup']['ownerID'] = projOwnerID
        self.tools.writeConfFile(projectConfig)


    def buyLocal (self, projectConfig) :
        '''Change the ownership on a local project by assigning your
        userID to it.'''

        projOwnerID = self.userConfig['System']['userID']
        projectConfig['Backup']['ownerID'] = projOwnerID
        self.tools.writeConfFile(projectConfig)


    def replaceProject (self, source, target) :
        '''This will completly replace an existing project (target)
        with data from another project (source). This assumes
        source and target are valid.'''

        # We simply just get rid of the target before doing a merge
        shutil.rmtree(target)
        self.log.writeToLog(self.errorCodes['4150'], [target, source])
        self.mergeProjects(source, target)


    def mergeProjects(self, source, target) :
        '''This will merge two Rapuma projects and try to preserve
        data in the target that is newer than the source. This assumes
        target and source are valid.'''
        
        # Get a list of files we do not want
        excludeFiles        = self.makeExcludeFileList(source)

        # Get a total list of files from the project
        cn = 0
        cr = 0
        # Add space for output message
        sys.stdout.write('\n')
        sys.stdout.write('Merging files from: ' + source + ' to: ' + target)
        sys.stdout.flush()
        for folder, subs, files in os.walk(source):
            for fileName in files:
                # Do not include any backup files we find
                if fileName[-1] == '~' :
                    continue
                if os.path.join(folder, fileName) not in excludeFiles :
                    if not os.path.isdir(folder.replace(source, target)) :
                        os.makedirs(folder.replace(source, target))
                    targetFile = os.path.join(folder, fileName).replace(source, target)
                    sourceFile = os.path.join(folder, fileName)
                    if not os.path.isfile(targetFile) :
                        sys.stdout.write('.')
                        sys.stdout.flush()
                        shutil.copy(sourceFile, targetFile)
                        cn +=1
                    # Otherwise if the cloud file is older than
                    # the project file, refresh it
                    elif self.tools.isOlder(targetFile, sourceFile) :
                        if os.path.isfile(targetFile) :
                            os.remove(targetFile)
                        sys.stdout.write('.')
                        sys.stdout.flush()
                        shutil.copy(sourceFile, targetFile)
                        cr +=1
        # Add space for next message
        sys.stdout.write('\n')


        # Report what happened
        self.log.writeToLog(self.errorCodes['4110'])
        if cn == 0 and cr == 0 :
            self.log.writeToLog(self.errorCodes['4120'])
        else :
            if cn > 0 :
                self.log.writeToLog(self.errorCodes['4130'], [str(cn)])
            if cr > 0 :
                self.log.writeToLog(self.errorCodes['4140'], [str(cr)])

        return True


    def getProjHome (self, tPath = None) :
        '''Return a project home path by checking to see what the best path
        might be. Provided path gets first dibs, then '''

        if tPath :
            if os.path.isfile(tPath) :
                return self.local.projHome
            elif self.tools.resolvePath(tPath) :
                tPath = self.tools.resolvePath(tPath)
                lastFolder = os.path.basename(tPath)
                if lastFolder == self.pid :
                    return tPath
                else :
                    return os.path.join(tPath, self.pid)
            else :
                self.log.writeToLog(self.errorCodes['4220'], [tPath])
        elif self.local.projHome :
            return self.local.projHome
        else :
            return self.tools.resolvePath(os.path.join(self.userConfig['Resources']['projects'], self.pid))
Example #2
0
class Usfm (Group) :
    '''This class contains information about a type of component 
    used in a type of project.'''

    # Shared values
    xmlConfFile     = 'usfm.xml'

    def __init__(self, project, cfg) :
        super(Usfm, self).__init__(project, cfg)

#        import pdb; pdb.set_trace()

        # Set values for this manager
        self.pid                    = project.projectIDCode
        self.gid                    = project.gid
        self.cType                  = 'usfm'
        self.Ctype                  = self.cType.capitalize()
        self.project                = project
        self.local                  = project.local
        self.tools                  = Tools()
        self.proj_font              = ProjFont(self.pid)
        self.proj_illustration      = ProjIllustration(self.pid, self.gid)
        self.proj_config            = Config(self.pid, self.gid)
        self.proj_config.getProjectConfig()
        self.proj_config.getAdjustmentConfig()
        self.projectConfig          = self.proj_config.projectConfig
        self.adjustmentConfig       = self.proj_config.adjustmentConfig
        self.log                    = project.log
        self.cfg                    = cfg
        self.mType                  = project.projectMediaIDCode
        self.renderer               = project.projectConfig['CompTypes'][self.Ctype]['renderer']
        self.sourceEditor           = project.projectConfig['CompTypes'][self.Ctype]['sourceEditor']
        self.macPackId              = project.projectConfig['CompTypes'][self.Ctype]['macroPackage']
        # Get the comp settings
        self.compSettings           = project.projectConfig['CompTypes'][self.Ctype]
        # Build a tuple of managers this component type needs to use
        self.usfmManagers = ('text', self.renderer)

        # Init the general managers
        for self.mType in self.usfmManagers :
            self.project.createManager(self.mType)

        # Create the internal ref names we use in this module
        self.text                   = self.project.managers[self.cType + '_Text']
        # File names

        # Folder paths
        self.projScriptFolder       = self.local.projScriptFolder
        self.projComponentFolder    = self.local.projComponentFolder
        self.gidFolder              = os.path.join(self.projComponentFolder, self.gid)
        # File names with folder paths
        self.rapumaXmlCompConfig    = os.path.join(self.project.local.rapumaConfigFolder, self.xmlConfFile)



        # Get persistant values from the config if there are any
        newSectionSettings = self.tools.getPersistantSettings(self.projectConfig['CompTypes'][self.Ctype], self.rapumaXmlCompConfig)
        if newSectionSettings != self.projectConfig['CompTypes'][self.Ctype] :
            self.projectConfig['CompTypes'][self.Ctype] = newSectionSettings
        # Set them here
        for k, v in self.compSettings.iteritems() :
            setattr(self, k, v)

        # Module Error Codes
        self.errorCodes     = {

            #'USFM-000' : ['MSG', 'Messages for the USFM module.'],
            #'USFM-005' : ['MSG', 'Unassigned error message ID.'],
            #'USFM-010' : ['ERR', 'Could not process character pair. This error was found: [<<1>>]. Process could not complete. - usfm.pt_tools.getNWFChars()'],
            #'USFM-020' : ['ERR', 'Improper character pair found: [<<1>>].  Process could not complete. - usfm.pt_tools.getNWFChars()'],
            #'USFM-025' : ['WRN', 'No non-word-forming characters were found in the PT settings file. - usfm.pt_tools.getNWFChars()'],
            #'USFM-040' : ['ERR', 'Hyphenation source file not found: [<<1>>]. Process halted!'],
            #'USFM-080' : ['LOG', 'Normalizing Unicode text to the [<<1>>] form.'],
            #'USFM-090' : ['ERR', 'USFM file: [<<1>>] did NOT pass the validation test. Because of an encoding conversion, the terminal output is from the file [<<2>>]. Please only edit [<<1>>].'],
            #'USFM-095' : ['WRN', 'Validation for USFM file: [<<1>>] was turned off.'],
            #'USFM-100' : ['MSG', 'Source file editor [<<1>>] is not recognized by this system. Please double check the name used for the source text editor setting.'],
            #'USFM-110' : ['ERR', 'Source file name could not be built because the Name Form ID for [<<1>>] is missing or incorrect. Double check to see which editor created the source text.'],
            #'USFM-120' : ['ERR', 'Source file: [<<1>>] not found! Cannot copy to project. Process halting now.'],
            #'USFM-130' : ['ERR', 'Failed to complete preprocessing on component [<<1>>]'],
            #'USFM-140' : ['MSG', 'Completed installation on [<<1>>] component working text.'],
            #'USFM-150' : ['ERR', 'Unable to copy [<<1>>] to [<<2>>] - error in text.'],

            '0010' : ['LOG', 'Created the [<<1>>] master adjustment file.'],
            '0220' : ['ERR', 'Cannot find: [<<1>>] working file, unable to complete preprocessing for rendering.'],
            '0230' : ['LOG', 'Created the [<<1>>] component adjustment file.'],
            '0240' : ['LOG', 'Could not find adjustments section for [<<1>>], created place holder setting.'],
            '0245' : ['LOG', 'Could not find adjustments for [<<1>>]. No ajustment file has been output.'],
            '0255' : ['LOG', 'Illustrations not being used. The piclist file has been removed from the [<<1>>] illustrations folder.'],
            '0260' : ['LOG', 'Piclist file for [<<1>>] has been created.'],
            '0265' : ['ERR', 'Failed to create piclist file for [<<1>>]!'],
            '0300' : ['ERR', 'One or more illustration files are missing from the project. Please import these files before continuing.']
        }


###############################################################################
############################ Functions Begin Here #############################
###############################################################################
######################## Error Code Block Series = 0200 #######################
###############################################################################

    def makeFileName(self, cid) :
        '''From what we know, return the full file name.'''

        # FIXME: We default this to "base" but for a diglot implementation
        # this is not going to work because we need to have a second
        # file name. Cross that bridge...

        return cid + '_base'


    def makeFileNameWithExt(self, cid) :
        '''From what we know, return the full file name.'''

        return self.makeFileName(cid) + '.' + self.cType


    def getCidPath (self, cid) :
        '''Return the full path of the cName working text file. This assumes
        the cid is valid.'''

        return os.path.join(self.local.projComponentFolder, cid, self.makeFileNameWithExt(cid))


    def getCidAdjPath (self, cid) :
        '''Return the full path of the cName working text adjustments file. 
        This assumes the cName is valid. Note that all macro packages that have
        a manual adjustment feature must use this naming scheme. The name syntax
        comes from the "mother" macro package which is ptx2pdf.'''

        return os.path.join(self.local.projComponentFolder, cid, self.makeFileNameWithExt(cid) + '.adj')


    def render(self, gid, cidList, pages, override, save) :
        '''Does USFM specific rendering of a USFM component'''

#        import pdb; pdb.set_trace()

        # If the whole group is being rendered, we need to preprocess it
        cids = []
        if not cidList :
            cids = self.projectConfig['Groups'][gid]['cidList']
        else :
            cids = cidList

        # Preprocess all subcomponents (one or more)
        # Stop if it breaks at any point
        for cid in cids :
            if not self.preProcessGroup(gid, [cid]) :
                return False

        # With everything in place we can render the component.
        # Note: We pass the cidList straight through
        self.project.managers['usfm_' + self.renderer.capitalize()].run(gid, cidList, pages, override, save)

        return True


    def preProcessGroup (self, gid, cidList) :
        '''This will prepare a component group for rendering by checking for
        and/or creating any dependents it needs to render properly.'''

#        import pdb; pdb.set_trace()

        # Get some relevant settings
        # FIXME: Note page border has not really been implemented yet.
        # It is different from backgound management
        useIllustrations        = self.tools.str2bool(self.projectConfig['Groups'][gid]['useIllustrations'])
        useManualAdjustments    = self.tools.str2bool(self.projectConfig['Groups'][gid]['useManualAdjustments'])

        # See if the working text is present for each subcomponent in the
        # component and try to install it if it is not
        for cid in cidList :
            cType = self.cfg['cType']
            cidUsfm = self.getCidPath(cid)
            # Test for source here and die if it isn't there
            if not os.path.isfile(cidUsfm) :
                self.log.writeToLog(self.errorCodes['0220'], [cidUsfm], 'usfm.preProcessGroup():0220')
            # Add/manage the dependent files for this cid

# FIXME: Some changes may be needed here to guide creation of adjustment files
            # Component adjustment file
            cidAdjFile = self.getCidAdjPath(cid)
            if useManualAdjustments :
                self.createCompAdjustmentFile(cid)
            else :
                # If no adjustments, remove any exsiting file
                if os.path.isfile(cidAdjFile) :
                    os.remove(cidAdjFile)
            # Component piclist file
            cidPiclistFile = self.proj_illustration.getCidPiclistFile(cid)
            if useIllustrations :
                if self.proj_illustration.hasIllustrations(cid) :
                    # Check for missing illustrations (die here if not found)
                    if self.proj_illustration.missingIllustrations(cid) :
                        self.log.writeToLog(self.errorCodes['0300'])
                    # Create piclist file if not there or if the config has changed
                    if not os.path.isfile(cidPiclistFile) or self.tools.isOlder(cidPiclistFile, self.local.illustrationConfFile) :
                        # Now make a fresh version of the piclist file
                        if self.proj_illustration.createPiclistFile(cid) :
                            self.log.writeToLog(self.errorCodes['0260'], [cid])
                        else :
                            self.log.writeToLog(self.errorCodes['0265'], [cid])
                    else :
                        for f in [self.local.layoutConfFile, self.local.illustrationConfFile] :
                            if self.tools.isOlder(cidPiclistFile, f) or not os.path.isfile(cidPiclistFile) :
                                # Remake the piclist file
                                if self.proj_illustration.createPiclistFile(cid) :
                                    self.log.writeToLog(self.errorCodes['0260'], [cid])
                                else :
                                    self.log.writeToLog(self.errorCodes['0265'], [cid])
                else :
                    # Does not seem to be any illustrations for this cid
                    # clean out any piclist file that might be there
                    if os.path.isfile(cidPiclistFile) :
                        os.remove(cidPiclistFile)
            else :
                # If we are not using illustrations then any existing piclist file will be removed
                if os.path.isfile(cidPiclistFile) :
                    os.remove(cidPiclistFile)
                    self.log.writeToLog(self.errorCodes['0255'], [cid])

        # Any more stuff to run?

        return True

# FIXME: Moved this to xetex.py as that was the only place it was called from
    #def checkStartPageNumber (self) :
        #'''Adjust page number for the current group. The current logic is
        #if there is no number in the startPageNumber setting, we can put
        #one in there as a suggestion. If there is already one there, the
        #user will be responsible for seeing that it is correct.'''

##        import pdb; pdb.set_trace()

        #try :
            ## Simply try to return anything that is in the field
            #cStrPgNo = self.projectConfig['Groups'][self.gid]['startPageNumber']
            #if cStrPgNo != '' :
                #return cStrPgNo
        #except :
            ## If nothing is there, we'll make a suggestion
            #pGrp = str(self.projectConfig['Groups'][self.gid]['precedingGroup'])
            #if pGrp == 'None' :
                #self.projectConfig['Groups'][self.gid]['startPageNumber'] = 1
                #self.tools.writeConfFile(self.projectConfig)
                #return '1'
            #else :
                ## Calculate the suggested number based on the preceeding group
                #try :
                    #cStrPgNo    = str(self.projectConfig['Groups'][self.gid]['startPageNumber'])
                #except :
                    #cStrPgNo    = 1
                    #self.projectConfig['Groups'][self.gid]['startPageNumber'] = 1
                #try :
                    #pGrpPgs     = int(self.projectConfig['Groups'][pGrp]['totalPages'])
                    #pGrpStrPgNo = int(self.projectConfig['Groups'][pGrp]['startPageNumber'])
                #except :
                    ## FIXME: Maybe this could go out and find out exactly how many pages were in the preceeding group
                    #pGrpPgs     = 1
                    #pGrpStrPgNo = 1
                    #self.projectConfig['Groups'][pGrp]['totalPages'] = 1
                    #self.projectConfig['Groups'][pGrp]['startPageNumber'] = 1
                ## Whether this is right or wrong set it the way it is
                #self.projectConfig['Groups'][self.gid]['startPageNumber'] = (pGrpStrPgNo + pGrpPgs)
                #self.tools.writeConfFile(self.projectConfig)
                #return self.projectConfig['Groups'][pGrp]['startPageNumber']


    def createCompAdjustmentFile (self, cid) :
        '''Create an adjustment file for this cid. If entries exsist in
        the adjustment.conf file.'''

        description = 'Auto-generated text adjustments file for: ' + cid + '\n'

#        import pdb; pdb.set_trace()

        # Check for a master adj conf file
        if os.path.exists(self.local.adjustmentConfFile) :
            adjFile = self.getCidAdjPath(cid)
            # Clean up old file if there is one so we can start fresh
            if os.path.exists(adjFile) :
                os.remove(adjFile)
            # Nothing to do if no gid section is found
            if not self.adjustmentConfig.has_key(self.gid) :
                self.tools.buildConfSection(self.adjustmentConfig, self.gid)
            if not self.adjustmentConfig[self.gid].has_key(cid) :
                self.tools.buildConfSection(self.adjustmentConfig[self.gid], cid)
                self.adjustmentConfig[self.gid][cid]['%1.1'] = '1'
                self.tools.writeConfFile(self.adjustmentConfig)
                self.log.writeToLog(self.errorCodes['0240'], [cid])
                return False
            # Sort through commented adjustment lines ()
            if self.adjustmentConfig[self.gid].has_key(cid) :
                c = False
                for k in self.adjustmentConfig[self.gid][cid].keys() :
                    if not re.search(r'%|#', k) :
                        c = True
                if not c :
                    self.log.writeToLog(self.errorCodes['0245'], [cid])
                    return False
            # If we make it this far, create the new adjustment file
            with codecs.open(adjFile, "w", encoding='utf_8') as writeObject :
                writeObject.write(self.tools.makeFileHeader(adjFile, description, True))
                # Output like this: JAS 1.13 +1
                for k, v in self.adjustmentConfig[self.gid][cid].iteritems() :
                    if re.search(r'%|#', k) :
                        continue
                    adj = v
                    if int(v) > 0 : 
                        adj = '+' + str(v)
                    writeObject.write(cid.upper() + ' ' + k + ' ' + adj + '\n')

                self.log.writeToLog(self.errorCodes['0230'], [self.tools.fName(adjFile)])

            return True


    def createProjAdjustmentConfFile (self) :
        '''Create a project master component adjustment file that group component
        ajustment files will be created automatically from. This will run every 
        time preprocess is run but after the first time it will only add a sections
        for new groups or components.'''

        if not os.path.exists(self.adjustmentConfFile) :
            self.adjustmentConfig = ConfigObj(self.adjustmentConfFile, encoding='utf-8')
            self.adjustmentConfig.filename = self.adjustmentConfFile
            self.updateCompAdjustmentConf()
        return True


    def updateCompAdjustmentConf (self) :
        '''Update an adjustmentConfig based on changes in the projectConfig.'''

        for gid in self.projectConfig['Groups'].keys() :
            if gid not in self.adjustmentConfig.keys() :
                self.tools.buildConfSection(self.adjustmentConfig, gid)
            for comp in self.projectConfig['Groups'][gid]['cidList'] :
                if not self.adjustmentConfig[gid].has_key(comp) :
                    self.tools.buildConfSection(self.adjustmentConfig[gid], comp)
                self.adjustmentConfig[gid][comp]['%1.1'] = '1'
        self.tools.writeConfFile(self.adjustmentConfig)
        return True


###############################################################################
######################## USFM Component Text Functions ########################
###############################################################################
######################## Error Code Block Series = 0400 #######################
###############################################################################


    def getComponentType (self, gid) :
        '''Return the cType for a component.'''

#        import pdb; pdb.set_trace()

        try :
            cType = self.projectConfig['Groups'][gid]['cType']
        except Exception as e :
            # If we don't succeed, we should probably quite here
            self.log.writeToLog('COMP-200', ['Key not found ' + str(e)])
            self.tools.dieNow()

        return cType


    def isCompleteComponent (self, gid, cid) :
        '''A two-part test to see if a component has a config entry and a file.'''

        if self.hasCidFile(gid, cid) :
            return True


    def hasUsfmCidInfo (self, cid) :
        '''Return True if this cid is in the PT USFM cid info dictionary.'''

        if cid in self.usfmCidInfo().keys() :
            return True


    def hasCidFile (self, gid, cid) :
        '''Return True or False depending on if a working file exists 
        for a given cName.'''

        cType = self.projectConfig['Groups'][gid]['cType']
        return os.path.isfile(os.path.join(self.local.projComponentFolder, cid, cid + '.' + cType))


    def usfmCidInfo (self) :
        '''Return a dictionary of all valid information about USFMs used in PT. Note
        that a couple special non-standard IDs have been added at the top of the list.'''

    #            ID     Comp Name                               Comp ID                         PT ID  Chps
        return {
                '_z_' : ['USFM InternalCaller',                 'usfm_internal_caller',         '00',   0], 
                'gen' : ['Genesis',                             'genesis',                      '01',  50], 
                'exo' : ['Exodus',                              'exodus',                       '02',  40], 
                'lev' : ['Leviticus',                           'leviticus',                    '03',  27], 
                'num' : ['Numbers',                             'numbers',                      '04',  36], 
                'deu' : ['Deuteronomy',                         'deuteronomy',                  '05',  34], 
                'jos' : ['Joshua',                              'joshua',                       '06',  24], 
                'jdg' : ['Judges',                              'judges',                       '07',  21], 
                'rut' : ['Ruth',                                'ruth',                         '08',   4], 
                '1sa' : ['1 Samuel',                            '1_samuel',                     '09',  31], 
                '2sa' : ['2 Samuel',                            '2_samuel',                     '10',  24], 
                '1ki' : ['1 Kings',                             '1_kings',                      '11',  22], 
                '2ki' : ['2 Kings',                             '2_kings',                      '12',  25], 
                '1ch' : ['1 Chronicles',                        '1_chronicles',                 '13',  29], 
                '2ch' : ['2 Chronicles',                        '2_chronicles',                 '14',  36], 
                'ezr' : ['Ezra',                                'ezra',                         '15',  10], 
                'neh' : ['Nehemiah',                            'nehemiah',                     '16',  13], 
                'est' : ['Esther',                              'esther',                       '17',  10], 
                'job' : ['Job',                                 'job',                          '18',  42], 
                'psa' : ['Psalms',                              'psalms',                       '19', 150], 
                'pro' : ['Proverbs',                            'proverbs',                     '20',  31], 
                'ecc' : ['Ecclesiastes',                        'ecclesiastes',                 '21',  12], 
                'sng' : ['Song of Songs',                       'song_of_songs',                '22',   8], 
                'isa' : ['Isaiah',                              'isaiah',                       '23',  66], 
                'jer' : ['Jeremiah',                            'jeremiah',                     '24',  52], 
                'lam' : ['Lamentations',                        'lamentations',                 '25',   5], 
                'ezk' : ['Ezekiel',                             'ezekiel',                      '26',  48], 
                'dan' : ['Daniel',                              'daniel',                       '27',  12], 
                'hos' : ['Hosea',                               'hosea',                        '28',  14], 
                'jol' : ['Joel',                                'joel',                         '29',   3], 
                'amo' : ['Amos',                                'amos',                         '30',   9], 
                'oba' : ['Obadiah',                             'obadiah',                      '31',   1], 
                'jon' : ['Jonah',                               'jonah',                        '32',   4], 
                'mic' : ['Micah',                               'micah',                        '33',   7], 
                'nam' : ['Nahum',                               'nahum',                        '34',   3], 
                'hab' : ['Habakkuk',                            'habakkuk',                     '35',   3], 
                'zep' : ['Zephaniah',                           'zephaniah',                    '36',   3], 
                'hag' : ['Haggai',                              'haggai',                       '37',   2], 
                'zec' : ['Zechariah',                           'zechariah',                    '38',  14], 
                'mal' : ['Malachi',                             'malachi',                      '39',   4],
                'mat' : ['Matthew',                             'matthew',                      '41',  28], 
                'mrk' : ['Mark',                                'mark',                         '42',  16], 
                'luk' : ['Luke',                                'luke',                         '43',  24], 
                'jhn' : ['John',                                'john',                         '44',  21], 
                'act' : ['Acts',                                'acts',                         '45',  28], 
                'rom' : ['Romans',                              'romans',                       '46',  16], 
                '1co' : ['1 Corinthians',                       '1_corinthians',                '47',  16], 
                '2co' : ['2 Corinthians',                       '2_corinthians',                '48',  13], 
                'gal' : ['Galatians',                           'galatians',                    '49',   6], 
                'eph' : ['Ephesians',                           'ephesians',                    '50',   6], 
                'php' : ['Philippians',                         'philippians',                  '51',   4], 
                'col' : ['Colossians',                          'colossians',                   '52',   4], 
                '1th' : ['1 Thessalonians',                     '1_thessalonians',              '53',   5], 
                '2th' : ['2 Thessalonians',                     '2_thessalonians',              '54',   3], 
                '1ti' : ['1 Timothy',                           '1_timothy',                    '55',   6], 
                '2ti' : ['2 Timothy',                           '2_timothy',                    '56',   4], 
                'tit' : ['Titus',                               'titus',                        '57',   3], 
                'phm' : ['Philemon',                            'philemon',                     '58',   1], 
                'heb' : ['Hebrews',                             'hebrews',                      '59',  13], 
                'jas' : ['James',                               'james',                        '60',   5], 
                '1pe' : ['1 Peter',                             '1_peter',                      '61',   5], 
                '2pe' : ['2 Peter',                             '2_peter',                      '62',   3], 
                '1jn' : ['1 John',                              '1_john',                       '63',   5], 
                '2jn' : ['2 John',                              '2_john',                       '64',   1], 
                '3jn' : ['3 John',                              '3_john',                       '65',   1], 
                'jud' : ['Jude',                                'jude',                         '66',   1], 
                'rev' : ['Revelation',                          'revelation',                   '67',  22], 
                'tob' : ['Tobit',                               'tobit',                        '68', '?'], 
                'jdt' : ['Judith',                              'judith',                       '69', '?'], 
                'esg' : ['Esther',                              'esther',                       '70', '?'], 
                'wis' : ['Wisdom of Solomon',                   'wisdom_of_solomon',            '71', '?'], 
                'sir' : ['Sirach',                              'sirach',                       '72', '?'], 
                'bar' : ['Baruch',                              'baruch',                       '73', '?'], 
                'lje' : ['Letter of Jeremiah',                  'letter_of_jeremiah',           '74', '?'], 
                's3y' : ['Song of the Three Children',          'song_3_children',              '75', '?'], 
                'sus' : ['Susanna',                             'susanna',                      '76', '?'], 
                'bel' : ['Bel and the Dragon',                  'bel_dragon',                   '77', '?'], 
                '1ma' : ['1 Maccabees',                         '1_maccabees',                  '78', '?'], 
                '2ma' : ['2 Maccabees',                         '2_maccabees',                  '79', '?'], 
                '3ma' : ['3 Maccabees',                         '3_maccabees',                  '80', '?'], 
                '4ma' : ['4 Maccabees',                         '4_maccabees',                  '81', '?'], 
                '1es' : ['1 Esdras',                            '1_esdras',                     '82', '?'], 
                '2es' : ['2 Esdras',                            '2_esdras',                     '83', '?'], 
                'man' : ['Prayer of Manasses',                  'prayer_of_manasses',           '84', '?'], 
                'ps2' : ['Psalms 151',                          'psalms_151',                   '85', '?'], 
                'oda' : ['Odae',                                'odae',                         '86', '?'], 
                'pss' : ['Psalms of Solomon',                   'psalms_of_solomon',            '87', '?'], 
                'jsa' : ['Joshua A',                            'joshua_a',                     '88', '?'], 
                'jdb' : ['Joshua B',                            'joshua_b',                     '89', '?'], 
                'tbs' : ['Tobit S',                             'tobit_s',                      '90', '?'], 
                'sst' : ['Susannah (Theodotion)',               'susannah_t',                   '91', '?'], 
                'dnt' : ['Daniel (Theodotion)',                 'daniel_t',                     '92', '?'], 
                'blt' : ['Bel and the Dragon (Theodotion)',     'bel_dragon_t',                 '93', '?'], 
                'frt' : ['Front Matter',                        'front_matter',                 'A0',   0], 
                'int' : ['Introductions',                       'introductions',                'A7',   0], 
                'bak' : ['Back Matter',                         'back_matter',                  'A1',   0], 
                'cnc' : ['Concordance',                         'concordance',                  'A8',   0], 
                'glo' : ['Glossary',                            'glossary',                     'A9',   0], 
                'tdx' : ['Topical Index',                       'topical_index',                'B0',   0], 
                'ndx' : ['Names Index',                         'names_index',                  'B1',   0], 
                'xxa' : ['Extra A',                             'extra_a',                      '94',   0], 
                'xxb' : ['Extra B',                             'extra_b',                      '95',   0], 
                'xxc' : ['Extra C',                             'extra_c',                      '96',   0], 
                'xxd' : ['Extra D',                             'extra_d',                      '97',   0],
                'xxe' : ['Extra E',                             'extra_e',                      '98',   0], 
                'xxf' : ['Extra F',                             'extra_f',                      '99',   0], 
                'xxg' : ['Extra G',                             'extra_g',                      '100',  0], 
                'oth' : ['Other',                               'other',                        'A2',   0], 
                'eza' : ['Apocalypse of Ezra',                  'apocalypse_of_ezra',           'A4', '?'], 
                '5ez' : ['5 Ezra',                              '5_ezra_lp',                    'A5', '?'], 
                '6ez' : ['6 Ezra (Latin Epilogue)',             '6_ezra_lp',                    'A6', '?'], 
                'dag' : ['Daniel Greek',                        'daniel_greek',                 'B2', '?'], 
                'ps3' : ['Psalms 152-155',                      'psalms_152-155',               'B3', '?'], 
                '2ba' : ['2 Baruch (Apocalypse)',               '2_baruch_apocalypse',          'B4', '?'], 
                'lba' : ['Letter of Baruch',                    'letter_of_baruch',             'B5', '?'], 
                'jub' : ['Jubilees',                            'jubilees',                     'B6', '?'], 
                'eno' : ['Enoch',                               'enoch',                        'B7', '?'], 
                '1mq' : ['1 Meqabyan',                          '1_meqabyan',                   'B8', '?'], 
                '2mq' : ['2 Meqabyan',                          '2_meqabyan',                   'B9', '?'], 
                '3mq' : ['3 Meqabyan',                          '3_meqabyan',                   'C0', '?'], 
                'rep' : ['Reproof (Proverbs 25-31)',            'reproof_proverbs_25-31',       'C1', '?'], 
                '4ba' : ['4 Baruch (Rest of Baruch)',           '4_baruch',                     'C2', '?'], 
                'lao' : ['Laodiceans',                          'laodiceans',                   'C3', '?'] 

               }