Exemple #1
0
def check_storage():
    EL_root = etree.Element('needcopies')    
    Policy_obj_list = ESSArchPolicy.objects.filter(PolicyStat=1).all()
    for Policy_obj in Policy_obj_list:
        sm_obj_list = [[Policy_obj.sm_1, Policy_obj.sm_type_1, Policy_obj.sm_format_1, Policy_obj.sm_blocksize_1, 
                        Policy_obj.sm_maxCapacity_1, Policy_obj.sm_minChunkSize_1, Policy_obj.sm_minContainerSize_1, 
                        Policy_obj.sm_target_1],
                       [Policy_obj.sm_2, Policy_obj.sm_type_2, Policy_obj.sm_format_2, Policy_obj.sm_blocksize_2, 
                        Policy_obj.sm_maxCapacity_2, Policy_obj.sm_minChunkSize_2, Policy_obj.sm_minContainerSize_2, 
                        Policy_obj.sm_target_2],
                       [Policy_obj.sm_3, Policy_obj.sm_type_3, Policy_obj.sm_format_3, Policy_obj.sm_blocksize_3, 
                        Policy_obj.sm_maxCapacity_3, Policy_obj.sm_minChunkSize_3, Policy_obj.sm_minContainerSize_3, 
                        Policy_obj.sm_target_3],
                       [Policy_obj.sm_4, Policy_obj.sm_type_4, Policy_obj.sm_format_4, Policy_obj.sm_blocksize_4, 
                        Policy_obj.sm_maxCapacity_4, Policy_obj.sm_minChunkSize_4, Policy_obj.sm_minContainerSize_4, 
                        Policy_obj.sm_target_4],
                       ]
        ip_obj_list = ArchiveObject.objects.filter(PolicyId=Policy_obj.PolicyID, StatusProcess=3000, StatusActivity=0).all()
        for ip_obj in ip_obj_list:
            storage_obj_list = ip_obj.storage_set.all()
            sm_num = 0 
            for sm_obj in sm_obj_list:
                sm_num += 1
                if sm_obj[0] == 1:
                    storage_count = 0
                    storageMediumID_list = []
                    for storage_obj in storage_obj_list:
                        storageMedium_obj = storage_obj.storageMediumUUID
                        if str(sm_obj[1])[0] == '2': #Disk
                            if storageMedium_obj.storageMedium == sm_obj[1] and storageMedium_obj.storageMediumID == 'disk':
                                storage_count+=1
                                storageMediumID_list.append(storageMedium_obj.storageMediumID)
                        elif str(sm_obj[1])[0] == '3': #Tape
                            if storageMedium_obj.storageMedium == sm_obj[1] and storageMedium_obj.storageMediumID.startswith(sm_obj[7]):
                                storage_count+=1
                                storageMediumID_list.append(storageMedium_obj.storageMediumID)
                    if storage_count == 0:
                        EL_object = etree.SubElement(EL_root, 'object', attrib={'id':ip_obj.ObjectIdentifierValue,
                                                                                'target':sm_obj[7],
                                                                                })
                        print 'Missing storage entry for storage method number: %s, target: %s, for object: %s' % ( sm_num, sm_obj[7], ip_obj.ObjectIdentifierValue)
                        
                    elif storage_count == 1:
                        print 'Found storage entry for storage method number: %s, target: %s (%s), for object: %s' % ( sm_num, sm_obj[7], ','.join(str(e) for e in storageMediumID_list), ip_obj.ObjectIdentifierValue)
                    else:
                        print 'Warning found to many storage entry for storage method number: %s, target: %s (%s), for object: %s' % ( sm_num, sm_obj[7], ','.join(str(e) for e in storageMediumID_list), ip_obj.ObjectIdentifierValue)
    doc = etree.ElementTree(element=EL_root, file=None)
    ESSMD.writeToFile(doc,'/ESSArch/log/needcopies/needcopies.xml')
Exemple #2
0
 def text2blob(self):
     #DOC,errno,why = ESSMETS.parseMetsFromFile('/ESSArch/bin/src/testdata/X0000001_METS.xml')
     DOC,errno,why = ESSMD.parseFromFile('/ESSArch/bin/src/testdata/X0000001_METS.xml')
     if errno:
         print why
     METSstr = etree.tostring(DOC,encoding='UTF-8', xml_declaration=True, pretty_print=True)
     blob = MySQLdb.escape_string(METSstr)
     print 'METSstr',METSstr
     print 'blob',blob
     #self.table = 'IngestObjectMetadata'
     ArchiveObjectMetadata_obj = ArchiveObjectMetadata()
     ArchiveObjectMetadata_obj.ObjectIdentifierValue = ObjectIdentifierValue
     ArchiveObjectMetadata_obj.ObjectMetadataType = 26
     ArchiveObjectMetadata_obj.ObjectMetadataBLOB = blob
     ArchiveObjectMetadata_obj.save() 
Exemple #3
0
 def text2blob(self):
     #DOC,errno,why = ESSMETS.parseMetsFromFile('/ESSArch/bin/src/testdata/X0000001_METS.xml')
     DOC, errno, why = ESSMD.parseFromFile(
         '/ESSArch/bin/src/testdata/X0000001_METS.xml')
     if errno:
         print why
     METSstr = etree.tostring(DOC,
                              encoding='UTF-8',
                              xml_declaration=True,
                              pretty_print=True)
     blob = MySQLdb.escape_string(METSstr)
     print 'METSstr', METSstr
     print 'blob', blob
     #self.table = 'IngestObjectMetadata'
     ArchiveObjectMetadata_obj = ArchiveObjectMetadata()
     ArchiveObjectMetadata_obj.ObjectIdentifierValue = ObjectIdentifierValue
     ArchiveObjectMetadata_obj.ObjectMetadataType = 26
     ArchiveObjectMetadata_obj.ObjectMetadataBLOB = blob
     ArchiveObjectMetadata_obj.save()
Exemple #4
0
    def ThreadMain(self,ProcName):
        logging.info('Starting ' + ProcName)
        self.tz=timezone.get_default_timezone()
        self.ChecksumAlgorithm_CHOICES_dict = dict(ChecksumAlgorithm_CHOICES)
        self.ChecksumAlgorithm_CHOICES_invdict = ESSPGM.Check().invert_dict(self.ChecksumAlgorithm_CHOICES_dict)
        while 1:
                if self.mDieFlag==1: break      # Request for death
                self.mLock.acquire()
                self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
                if self.Run == '0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                    self.RunFlag=0
                    self.mLock.release()
                    if Debug: logging.info('RunFlag: 0')
                    time.sleep(2)
                    continue
                # Process Item 
                lock=thread.allocate_lock()
                self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
                self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
                if ExtDBupdate:
                    self.ext_IngestTable = self.IngestTable
                else:
                    self.ext_IngestTable = ''
                #if Debug: logging.info('Start to list worklist (self.dbget)')
                self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue','ObjectPackageName',
                                                                                  'PolicyId','MetaObjectIdentifier',
                                                                                  'MetaObjectSize','DataObjectSize',
                                                                                  'ObjectSize',
                                                                                  'ObjectUUID'),
                                                                                 ('StatusActivity','=','0','AND',
                                                                                  'StatusProcess','BETWEEN',49,'AND',51))
                if errno: logging.error('Failed to access Local DB, error: ' + str(why))
                for self.obj in self.dbget:
                    self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                    if self.ProcDB[0]=='0':
                        logging.info('Stopping ' + ProcName)
                        ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                        thread.interrupt_main()
                        time.sleep(5)
                        break
                    elif self.ProcDB[1]==1:
                        while 1:
                            time.sleep(60)
                            self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                            if self.ProcDB[1]==1:
                                logging.info('Process is in pause state')
                            else:
                                break
                    self.ObjectIdentifierValue = self.obj[0]
                    self.ObjectPackageName = self.obj[1]
                    self.PolicyId = self.obj[2]
                    #self.MetaObjectIdentifier = self.obj[3] 	#METS filename
                    #self.MetaObjectSize = self.obj[4]		#METS size (bytes)
                    #self.DataObjectSize = self.obj[5]
                    self.ObjectSize = self.obj[6]
                    self.ObjectUUID = self.obj[7]
                    logging.debug('self.obj: '+str(self.obj))

                    self.ok = 1

                    self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','ChecksumAlgorithm','IngestPath','ValidateChecksum','ValidateXML'),('PolicyID',self.PolicyId))
                    if errno:
                        logging.error('Failed to access Local DB, error: ' + str(why))
                        self.ok = 0
                    if self.ok:
                        ###########################################################
                        # set variables
                        self.AIPpath = self.PolicyDB[0][0]
                        self.metatype = self.PolicyDB[0][1]
                        self.ChecksumAlgorithm = self.PolicyDB[0][2]
                        self.SIPpath = self.PolicyDB[0][3]
                        self.ValidateChecksum = self.PolicyDB[0][4]
                        self.ValidateXML = self.PolicyDB[0][5]
                        self.ObjectPath = os.path.join(self.AIPpath,self.ObjectPackageName)
                        self.Pmets_objpath = os.path.join(self.AIPpath,self.ObjectIdentifierValue + '_Package_METS.xml')

                    if self.metatype > 0:
                        self.startVerTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                        errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 50, 5)
                        if errno: logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                        logging.info('Start validate AIP package: ' + self.ObjectIdentifierValue)
                        self.Cmets_obj = None 
                        self.Cmets_objpath = None 
                        self.premis_obj = None 
                        self.premis_objpath = None
                        self.addml_obj = None
                        self.addml_objpath = None
                        ##########################################
                        # Get PMETS info
                        if self.ok:
                            [self.Package_info,self.CMets_info],errno,why = ESSMD.getPMETSInfo(FILENAME=self.Pmets_objpath)
                            if errno:
                                self.event_info = 'Failed to get PMETS info for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Object: %s, Package_info: %s, CMets_info: %s',self.ObjectIdentifierValue,str(self.Package_info),str(self.CMets_info))
                                # CMets_info and Package_info: ['A0007600_Content_METS.xml', 'MD5', 'b0270cb4d196b72b87fe27ce6242df18', 64058, 'text/xml']
                                #if self.Package_info[1] == 'MD5': self.PackageMessageDigestAlgorithm = 1
                                self.PackageMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[self.Package_info[1]]
                                if self.Package_info[2]: self.PackageMessageDigest = self.Package_info[2]
                                if self.Package_info[3]: self.PackageSize = int(self.Package_info[3])
                                self.Cmets_obj = self.CMets_info[0]
                                if self.metatype in [1,2,3]:
                                    self.Cmets_objpath = os.path.join(self.AIPpath,self.Cmets_obj)
                                elif self.metatype in [4]:
                                    self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
                                #if self.CMets_info[1] == 'MD5': self.CMetsMessageDigestAlgorithm = 1
                                self.CMetsMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[self.CMets_info[1]]
                                if self.CMets_info[2]: self.CMetsMessageDigest = self.CMets_info[2]
                                if self.CMets_info[3]: self.CMetsSize = int(self.CMets_info[3])
                        ##########################################################
                        # Check if ObjectPath and Cmets_objpath exist
                        if os.path.exists(self.ObjectPath) and os.path.exists(self.Cmets_objpath):
                            self.ok = 1
                        else:
                            self.event_info = 'The path to Object: %s or METS_Metaobject: %s is not accessible!' % (self.ObjectPath,self.Cmets_objpath)
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                            self.ok = 0
                        ##########################################
                        # Get MetsFgrp001TotalSize from METSfile 
                        if self.ok:
                            self.MetsFgrp001TotalSize,errno,why = ESSMD.getFileSizeFgrp001(FILENAME=self.Cmets_objpath)
                            if errno:
                                self.event_info = 'Failed to get MetsFgrp001TotalSize for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Object: %s, MetsFgrp001TotalSize: %s',self.ObjectIdentifierValue,str(self.MetsFgrp001TotalSize))
                        ##########################################
                        # Get PremisObjectTotalSize from METSfile
                        if self.ok:
                            if self.metatype in [1,2,3]:
                                self.premis_objpath = self.Cmets_objpath
                            elif self.metatype in [4]:
                                # Get metadata from METS file
                                res_info, res_files, res_struct, errno, why = ESSMD.getMETSFileList(FILENAME=self.Cmets_objpath)
                                if errno:
                                    self.event_info = 'Failed to get metadata from content METS for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    for res_file in res_files:
                                        if res_file[0] == 'amdSec' and res_file[2] == 'digiprovMD':
                                            self.premis_obj = res_file[8][5:]
                                            self.premis_objpath = '%s/%s/%s' % (self.SIPpath,self.ObjectIdentifierValue,self.premis_obj)
                                        elif res_file[0] == 'amdSec' and res_file[2] == 'techMD' and res_file[16] == 'ADDML': 
                                            self.addml_obj = res_file[8][5:]
                                            self.addml_objpath = '%s/%s/%s' % (self.SIPpath,self.ObjectIdentifierValue,self.addml_obj)
                            self.PremisObjectTotalSize,errno,why = ESSMD.getFileSizePremis(FILENAME=self.premis_objpath)
                            if errno:
                                self.event_info = 'Failed to get PREMISobjects total size in METSfile for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Object: %s, PremisObjectTotalSize: %s',self.ObjectIdentifierValue,str(self.PremisObjectTotalSize))
                        ###################################################
                        # Check if MetsFgrp001TotalSize is equal to PremisObjectTotalSize
                        if self.ok:
                            if self.MetsFgrp001TotalSize == self.PremisObjectTotalSize:
                                logging.info('Succeeded to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is equal to PremisObjectTotalSize: %s',self.ObjectIdentifierValue,str(self.MetsFgrp001TotalSize),str(self.PremisObjectTotalSize))
                            else:
                                self.event_info = 'Failed to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is not equal to PremisObjectTotalSize: %s' % (self.ObjectIdentifierValue,str(self.MetsFgrp001TotalSize),str(self.PremisObjectTotalSize))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                        ##########################################
                        # Get CMetsTotalSize
                        if self.ok:
                            self.CMetsTotalSize,errno,why = ESSMD.getTotalSize(FILENAME=self.Cmets_objpath)
                            if errno:
                                self.event_info = 'Failed to get CMetsTotalSize for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Object: %s, raw CMetsTotalSize: %s',self.ObjectIdentifierValue,str(self.CMetsTotalSize))
                                if self.CMetsSize:
                                    # Add Content Mets filesize
                                    self.CMetsTotalSize[0] += 1
                                    self.CMetsTotalSize[1] += self.CMetsSize
                                logging.debug('Object: %s, CMetsTotalSize: %s',self.ObjectIdentifierValue,str(self.CMetsTotalSize))
                        ##########################################
                        # Get TarFileSize
                        if self.ok:
                            self.TarFileSize,errno,why = ESSPGM.Check().getFileSizeTAR(self.ObjectPath)
                            if errno:
                                self.event_info = 'Failed to get TarFileSize for object: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Object: %s, TarFileSize: %s',self.ObjectIdentifierValue,str(self.TarFileSize))
                        ###################################################
                        # Check if CMetsTotalSize is equal to TarFileSize
                        if self.ok:
                            if self.CMetsTotalSize == self.TarFileSize:
                                logging.info('Succeeded to verify total package size for object: %s, CMetsTotalSize: %s is equal to TarFileSize: %s',self.ObjectIdentifierValue,str(self.CMetsTotalSize),str(self.TarFileSize))
                            else:
                                self.event_info = 'Failed to verify total package size for object: %s, CMetsTotalSize: %s is not equal to TarFileSize: %s' % (self.ObjectIdentifierValue,str(self.CMetsTotalSize),str(self.TarFileSize))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                        ########################
                        # get checksum for PMETS file
                        if self.ok:
                            self.PMetsMessageDigestAlgorithm = self.ChecksumAlgorithm
                            self.PMetsMessageDigest = ''
                            self.Pmets_obj_checksum,errno,why = ESSPGM.Check().checksum(self.Pmets_objpath, self.PMetsMessageDigestAlgorithm) # Checksum
                            if errno:
                                self.event_info = 'Failed to get checksum for: %s, Error: %s' % (self.Cmets_objpath,str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Checksum for PMETS file: %s',self.Pmets_obj_checksum)
                                self.PMetsMessageDigestAlgorithm = 1
                                self.PMetsMessageDigest = self.Pmets_obj_checksum
                        ########################
                        # get checksum for CMETS file
                        if self.ok:
                            self.Cmets_obj_checksum,errno,why = ESSPGM.Check().checksum(self.Cmets_objpath, self.CMetsMessageDigestAlgorithm) # Checksum
                            if errno:
                                self.event_info = 'Failed to get checksum for: %s, Error: %s' % (self.Cmets_objpath,str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.debug('Checksum for CMETS file: %s',self.Cmets_obj_checksum)
                        ###################################################
                        # Check CMetsMessageDigest 
                        if self.ok:
                            if self.ValidateChecksum:
                                if self.CMetsMessageDigest == self.Cmets_obj_checksum:
                                    logging.info('Succeeded to verify Content Mets MessageDigest for object: %s',self.ObjectIdentifierValue)
                                else:
                                    self.event_info = 'Failed to verify Content Mets MessageDigest for object: %s, CMetsMessageDigest: %s is equal to FileMessageDigest: %s' % (self.ObjectIdentifierValue,str(self.CMetsMessageDigest),str(self.Cmets_obj_checksum))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                            else:
                                logging.warning('Checksum validate is disabled for object: %s',self.ObjectIdentifierValue)
                        if self.ValidateXML:
                            ########################
                            # XML Schema Validate CMETS file
                            if self.ok:
                                errno,why = ESSMD.validate(FILENAME=self.Cmets_objpath)
                                if errno:
                                    self.event_info = 'Failed to schema validate Content METS file for object: %s, why: %s' % (self.ObjectIdentifierValue, str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info('Succeeded to schema validate Content METS file for object: %s',self.ObjectIdentifierValue)
                            ########################
                            # XML Schema Validate PMETS file
                            if self.ok:
                                errno,why = ESSMD.validate(FILENAME=self.Pmets_objpath)
                                if errno:
                                    self.event_info = 'Failed to schema validate Package METS file for object: %s, why: %s' % (self.ObjectIdentifierValue, str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info('Succeeded to schema validate Package METS file for object: %s',self.ObjectIdentifierValue)
                            ########################
                            # XML Schema Validate PREMIS file
                            if self.ok and self.premis_obj:
                                errno,why = ESSMD.validate(FILENAME=self.premis_objpath)
                                if errno:
                                    self.event_info = 'Failed to schema validate PREMIS file for object: %s, why: %s' % (self.ObjectIdentifierValue, str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info('Succeeded to schema validate PREMIS file for object: %s',self.ObjectIdentifierValue)
                            ########################
                            # XML Schema Validate ADDML file
                            if self.ok and self.addml_obj:
                                errno,why = ESSMD.validate(FILENAME=self.addml_objpath)
                                if errno:
                                    self.event_info = 'Failed to schema validate ADDML file for object: %s, why: %s' % (self.ObjectIdentifierValue, str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info('Succeeded to schema validate ADDML file for object: %s',self.ObjectIdentifierValue)
                        else:
                            logging.warning('Schema validate XML is disabled for object: %s',self.ObjectIdentifierValue)
                        self.stopVerTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                        self.VerTime = self.stopVerTime-self.startVerTime
                        if self.VerTime.seconds < 1: self.VerTime = datetime.timedelta(seconds=1)   #Fix min time to 1 second if it is zero.
                        self.ObjectSizeMB = int(self.ObjectSize)/1048576
                        self.VerMBperSEC = int(self.ObjectSizeMB)/int(self.VerTime.seconds)
                        ##################################
                        # Write metadatafiles to DB-blob or FTP
                        if self.ok:
                            if ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','MD_FTP_HOST'))[0][0]:
                                if self.Cmets_obj:
                                    ##################################
                                    # Write CMETS metadatafile to DB-blob
                                    res,errno,why = ESSmetablob.prod().StoreMetadataBlob(ObjectUUID=self.ObjectUUID,
                                                                                         ObjectIdentifierValue=self.ObjectIdentifierValue,
                                                                                         ObjectMetadataType=26,
                                                                                         FILENAME=self.Cmets_objpath,
                                                                                         FTPFileName=string.replace(self.Cmets_obj,self.ObjectIdentifierValue+'/',''),
                                                                                         FTPflag=1,
                                                                                         DBflag=0,
                                                                                         AgentIdentifierValue=AgentIdentifierValue)
                                    if errno:
                                        self.event_info = 'Failed to store Content METS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue, str(errno), str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                        self.ok = 0
                                    else:
                                        logging.info('Succeeded to store Content METS file to FTP server or DB-blob for object: %s',self.ObjectIdentifierValue)
                                if self.premis_obj:
                                    ##################################
                                    # Write PREMIS metadatafile to DB-blob
                                    res,errno,why = ESSmetablob.prod().StoreMetadataBlob(ObjectUUID=self.ObjectUUID,
                                                                                         ObjectIdentifierValue=self.ObjectIdentifierValue,
                                                                                         ObjectMetadataType=27,
                                                                                         FILENAME=self.premis_objpath,
                                                                                         FTPFileName=self.premis_obj,
                                                                                         FTPflag=1,
                                                                                         DBflag=0,
                                                                                         AgentIdentifierValue=AgentIdentifierValue)
                                    if errno:
                                        self.event_info = 'Failed to store PREMIS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue, str(errno), str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                        self.ok = 0
                                    else:
                                        logging.info('Succeeded to store PREMIS file to FTP server or DB-blob for object: %s',self.ObjectIdentifierValue)
                                if self.addml_obj:
                                    ##################################
                                    # Write ADDML metadatafile to DB-blob
                                    res,errno,why = ESSmetablob.prod().StoreMetadataBlob(ObjectUUID=self.ObjectUUID,
                                                                                         ObjectIdentifierValue=self.ObjectIdentifierValue,
                                                                                         ObjectMetadataType=25,
                                                                                         FILENAME=self.addml_objpath,
                                                                                         FTPFileName=self.addml_obj,
                                                                                         FTPflag=1,
                                                                                         DBflag=0,
                                                                                         AgentIdentifierValue=AgentIdentifierValue)
                                    if errno:
                                        self.event_info = 'Failed to store ADDML file to FTP server or DB-blob: %s, errno: %s, why: %s' % (self.ObjectIdentifierValue, str(errno), str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                        self.ok = 0
                                    else:
                                        logging.info('Succeeded to store ADDML file to FTP server or DB-blob for object: %s',self.ObjectIdentifierValue)
                            else:
                                logging.info('Skip to store metadata to FTP server or DB-blob for object: %s',self.ObjectIdentifierValue)
                        if not self.ok:
                            errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 51, 4)
                            if errno: 
                                logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                            else:
                                self.event_info = 'Failed to validate AIP package: ' + self.ObjectIdentifierValue
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                        else:
                            logging.info('Succeeded to validate AIP package: ' + self.ObjectIdentifierValue + ' , ' + str(self.VerMBperSEC) + ' MB/Sec and Time: ' + str(self.VerTime))
                            self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                            self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                            res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('StatusProcess','59',
                                                                                      'StatusActivity','0',
                                                                                      'CMetaMessageDigestAlgorithm',self.CMetsMessageDigestAlgorithm,
                                                                                      'CMetaMessageDigest',self.CMetsMessageDigest,
                                                                                      'PMetaMessageDigestAlgorithm',self.PMetsMessageDigestAlgorithm,
                                                                                      'PMetaMessageDigest',self.PMetsMessageDigest,
                                                                                      'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                                      'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                                      'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                                     ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                            if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                            else: ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'0','',2,self.ObjectIdentifierValue)
                            if errno == 0 and ExtDBupdate:
                                ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('StatusProcess','59',
                                                                                                         'StatusActivity','0',
                                                                                                         'CMetaMessageDigestAlgorithm',self.CMetsMessageDigestAlgorithm,
                                                                                                         'CMetaMessageDigest',self.CMetsMessageDigest,
                                                                                                         'PMetaMessageDigestAlgorithm',self.PMetsMessageDigestAlgorithm,
                                                                                                         'PMetaMessageDigest',self.PMetsMessageDigest,
                                                                                                         'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                                         'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                                        ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                                else:
                                    res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                    if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))

                    elif self.metatype == 0: #self.metatype 0 = No metadata
                        logging.info('Skip to validate AIP package: ' + self.ObjectIdentifierValue)
                        self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                        self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                        res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('StatusProcess','59',
                                                                                  'StatusActivity','0',
                                                                                  'CMetaMessageDigestAlgorithm','0',
                                                                                  'PMetaMessageDigestAlgorithm','0',
                                                                                  'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                                  'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                                  'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                                 ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                        if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                        else: ESSPGM.Events().create('1050','','ESSArch AIPValidate',ProcVersion,'0','Skip to validate AIP package',2,self.ObjectIdentifierValue)
                        if errno == 0 and ExtDBupdate:
                            ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('StatusProcess','59',
                                                                                                     'StatusActivity','0',
                                                                                                     'CMetaMessageDigestAlgorithm','0',
                                                                                                     'PMetaMessageDigestAlgorithm','0',
                                                                                                     'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                                     'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                                    ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                            if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                            else:
                                res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                db.close_old_connections()
                self.mLock.release()
                time.sleep(int(self.Time))
        self.mDieFlag=0
Exemple #5
0
    def ThreadMain(self, ProcName):
        logging.info('Starting ' + ProcName)
        self.tz = timezone.get_default_timezone()
        self.ChecksumAlgorithm_CHOICES_dict = dict(ChecksumAlgorithm_CHOICES)
        self.ChecksumAlgorithm_CHOICES_invdict = ESSPGM.Check().invert_dict(
            self.ChecksumAlgorithm_CHOICES_dict)
        while 1:
            if self.mDieFlag == 1: break  # Request for death
            self.mLock.acquire()
            self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
                                                    ('Time', 'Run'),
                                                    ('Name', ProcName))[0]
            if self.Run == '0':
                logging.info('Stopping ' + ProcName)
                ESSDB.DB().action('ESSProc', 'UPD',
                                  ('Status', '0', 'Run', '0', 'PID', '0'),
                                  ('Name', ProcName))
                self.RunFlag = 0
                self.mLock.release()
                if Debug: logging.info('RunFlag: 0')
                time.sleep(2)
                continue
            # Process Item
            lock = thread.allocate_lock()
            self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
                                                 ('Value', ),
                                                 ('Name', 'IngestTable'))[0][0]
            if ExtDBupdate:
                self.ext_IngestTable = self.IngestTable
            else:
                self.ext_IngestTable = ''
            #if Debug: logging.info('Start to list worklist (self.dbget)')
            self.dbget, errno, why = ESSDB.DB().action(
                self.IngestTable, 'GET4',
                ('ObjectIdentifierValue', 'ObjectPackageName', 'PolicyId',
                 'MetaObjectIdentifier', 'MetaObjectSize', 'DataObjectSize',
                 'ObjectSize', 'ObjectUUID'),
                ('StatusActivity', '=', '0', 'AND', 'StatusProcess', 'BETWEEN',
                 49, 'AND', 51))
            if errno:
                logging.error('Failed to access Local DB, error: ' + str(why))
            for self.obj in self.dbget:
                self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
                                                ('Run', 'Pause'),
                                                ('Name', ProcName))[0]
                if self.ProcDB[0] == '0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc', 'UPD',
                                      ('Status', '0', 'Run', '0', 'PID', '0'),
                                      ('Name', ProcName))
                    thread.interrupt_main()
                    time.sleep(5)
                    break
                elif self.ProcDB[1] == 1:
                    while 1:
                        time.sleep(60)
                        self.ProcDB = ESSDB.DB().action(
                            'ESSProc', 'GET', ('Run', 'Pause'),
                            ('Name', ProcName))[0]
                        if self.ProcDB[1] == 1:
                            logging.info('Process is in pause state')
                        else:
                            break
                self.ObjectIdentifierValue = self.obj[0]
                self.ObjectPackageName = self.obj[1]
                self.PolicyId = self.obj[2]
                #self.MetaObjectIdentifier = self.obj[3] 	#METS filename
                #self.MetaObjectSize = self.obj[4]		#METS size (bytes)
                #self.DataObjectSize = self.obj[5]
                self.ObjectSize = self.obj[6]
                self.ObjectUUID = self.obj[7]
                logging.debug('self.obj: ' + str(self.obj))

                self.ok = 1

                ArchivePolicy_obj = ArchivePolicy.objects.get(
                    PolicyStat=1, PolicyID=self.PolicyId)
                if self.ok:
                    ###########################################################
                    # set variables
                    self.AIPpath = ArchivePolicy_obj.AIPpath
                    self.metatype = ArchivePolicy_obj.IngestMetadata
                    self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
                    self.SIPpath = ArchivePolicy_obj.IngestPath
                    self.ValidateChecksum = ArchivePolicy_obj.ValidateChecksum
                    self.ValidateXML = ArchivePolicy_obj.ValidateXML
                    self.ObjectPath = os.path.join(self.AIPpath,
                                                   self.ObjectPackageName)
                    self.Pmets_objpath = os.path.join(
                        self.AIPpath,
                        self.ObjectIdentifierValue + '_Package_METS.xml')

                if self.metatype > 0:
                    self.startVerTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    errno, why = ESSPGM.DB().SetAIPstatus(
                        self.IngestTable, self.ext_IngestTable,
                        AgentIdentifierValue, self.ObjectUUID, 50, 5)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    logging.info('Start validate AIP package: ' +
                                 self.ObjectIdentifierValue)
                    self.Cmets_obj = None
                    self.Cmets_objpath = None
                    self.premis_obj = None
                    self.premis_objpath = None
                    self.addml_obj = None
                    self.addml_objpath = None
                    ##########################################
                    # Get PMETS info
                    if self.ok:
                        [self.Package_info,
                         self.CMets_info], errno, why = ESSMD.getPMETSInfo(
                             FILENAME=self.Pmets_objpath)
                        if errno:
                            self.event_info = 'Failed to get PMETS info for object: %s, errno: %s, why: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug(
                                'Object: %s, Package_info: %s, CMets_info: %s',
                                self.ObjectIdentifierValue,
                                str(self.Package_info), str(self.CMets_info))
                            # CMets_info and Package_info: ['A0007600_Content_METS.xml', 'MD5', 'b0270cb4d196b72b87fe27ce6242df18', 64058, 'text/xml']
                            #if self.Package_info[1] == 'MD5': self.PackageMessageDigestAlgorithm = 1
                            self.PackageMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[
                                self.Package_info[1]]
                            if self.Package_info[2]:
                                self.PackageMessageDigest = self.Package_info[
                                    2]
                            if self.Package_info[3]:
                                self.PackageSize = int(self.Package_info[3])
                            self.Cmets_obj = self.CMets_info[0]
                            if self.metatype in [1, 2, 3]:
                                self.Cmets_objpath = os.path.join(
                                    self.AIPpath, self.Cmets_obj)
                            elif self.metatype in [4]:
                                self.Cmets_objpath = os.path.join(
                                    self.SIPpath, self.Cmets_obj)
                            #if self.CMets_info[1] == 'MD5': self.CMetsMessageDigestAlgorithm = 1
                            self.CMetsMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[
                                self.CMets_info[1]]
                            if self.CMets_info[2]:
                                self.CMetsMessageDigest = self.CMets_info[2]
                            if self.CMets_info[3]:
                                self.CMetsSize = int(self.CMets_info[3])
                    ##########################################################
                    # Check if ObjectPath and Cmets_objpath exist
                    if os.path.exists(self.ObjectPath) and os.path.exists(
                            self.Cmets_objpath):
                        self.ok = 1
                    else:
                        self.event_info = 'The path to Object: %s or METS_Metaobject: %s is not accessible!' % (
                            self.ObjectPath, self.Cmets_objpath)
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1050', '',
                                               'ESSArch AIPValidate',
                                               ProcVersion, '1',
                                               self.event_info, 2,
                                               self.ObjectIdentifierValue)
                        self.ok = 0
                    ##########################################
                    # Get MetsFgrp001TotalSize from METSfile
                    if self.ok:
                        self.MetsFgrp001TotalSize, errno, why = ESSMD.getFileSizeFgrp001(
                            FILENAME=self.Cmets_objpath)
                        if errno:
                            self.event_info = 'Failed to get MetsFgrp001TotalSize for object: %s, errno: %s, why: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug(
                                'Object: %s, MetsFgrp001TotalSize: %s',
                                self.ObjectIdentifierValue,
                                str(self.MetsFgrp001TotalSize))
                    ##########################################
                    # Get PremisObjectTotalSize from METSfile
                    if self.ok:
                        if self.metatype in [1, 2, 3]:
                            self.premis_objpath = self.Cmets_objpath
                        elif self.metatype in [4]:
                            # Get metadata from METS file
                            res_info, res_files, res_struct, errno, why = ESSMD.getMETSFileList(
                                FILENAME=self.Cmets_objpath)
                            if errno:
                                self.event_info = 'Failed to get metadata from content METS for object: %s, errno: %s, why: %s' % (
                                    self.ObjectIdentifierValue, str(errno),
                                    str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                for res_file in res_files:
                                    if res_file[0] == 'amdSec' and res_file[
                                            2] == 'digiprovMD':
                                        self.premis_obj = res_file[8][5:]
                                        self.premis_objpath = '%s/%s/%s' % (
                                            self.SIPpath,
                                            self.ObjectIdentifierValue,
                                            self.premis_obj)
                                    elif res_file[0] == 'amdSec' and res_file[
                                            2] == 'techMD' and res_file[
                                                16] == 'ADDML':
                                        self.addml_obj = res_file[8][5:]
                                        self.addml_objpath = '%s/%s/%s' % (
                                            self.SIPpath,
                                            self.ObjectIdentifierValue,
                                            self.addml_obj)
                        self.PremisObjectTotalSize, errno, why = ESSMD.getFileSizePremis(
                            FILENAME=self.premis_objpath)
                        if errno:
                            self.event_info = 'Failed to get PREMISobjects total size in METSfile for object: %s, errno: %s, why: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug(
                                'Object: %s, PremisObjectTotalSize: %s',
                                self.ObjectIdentifierValue,
                                str(self.PremisObjectTotalSize))
                    ###################################################
                    # Check if MetsFgrp001TotalSize is equal to PremisObjectTotalSize
                    if self.ok:
                        if self.MetsFgrp001TotalSize == self.PremisObjectTotalSize:
                            logging.info(
                                'Succeeded to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is equal to PremisObjectTotalSize: %s',
                                self.ObjectIdentifierValue,
                                str(self.MetsFgrp001TotalSize),
                                str(self.PremisObjectTotalSize))
                        else:
                            self.event_info = 'Failed to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is not equal to PremisObjectTotalSize: %s' % (
                                self.ObjectIdentifierValue,
                                str(self.MetsFgrp001TotalSize),
                                str(self.PremisObjectTotalSize))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                    ##########################################
                    # Get CMetsTotalSize
                    if self.ok:
                        self.CMetsTotalSize, errno, why = ESSMD.getTotalSize(
                            FILENAME=self.Cmets_objpath)
                        if errno:
                            self.event_info = 'Failed to get CMetsTotalSize for object: %s, errno: %s, why: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug('Object: %s, raw CMetsTotalSize: %s',
                                          self.ObjectIdentifierValue,
                                          str(self.CMetsTotalSize))
                            if self.CMetsSize:
                                # Add Content Mets filesize
                                self.CMetsTotalSize[0] += 1
                                self.CMetsTotalSize[1] += self.CMetsSize
                            logging.debug('Object: %s, CMetsTotalSize: %s',
                                          self.ObjectIdentifierValue,
                                          str(self.CMetsTotalSize))
                    ##########################################
                    # Get TarFileSize
                    if self.ok:
                        self.TarFileSize, errno, why = ESSPGM.Check(
                        ).getFileSizeTAR(self.ObjectPath)
                        if errno:
                            self.event_info = 'Failed to get TarFileSize for object: %s, errno: %s, why: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug('Object: %s, TarFileSize: %s',
                                          self.ObjectIdentifierValue,
                                          str(self.TarFileSize))
                    ###################################################
                    # Check if CMetsTotalSize is equal to TarFileSize
                    if self.ok:
                        if self.CMetsTotalSize == self.TarFileSize:
                            logging.info(
                                'Succeeded to verify total package size for object: %s, CMetsTotalSize: %s is equal to TarFileSize: %s',
                                self.ObjectIdentifierValue,
                                str(self.CMetsTotalSize),
                                str(self.TarFileSize))
                        else:
                            self.event_info = 'Failed to verify total package size for object: %s, CMetsTotalSize: %s is not equal to TarFileSize: %s' % (
                                self.ObjectIdentifierValue,
                                str(self.CMetsTotalSize), str(
                                    self.TarFileSize))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                    ########################
                    # get checksum for PMETS file
                    if self.ok:
                        self.PMetsMessageDigestAlgorithm = self.ChecksumAlgorithm
                        self.PMetsMessageDigest = ''
                        self.Pmets_obj_checksum, errno, why = ESSPGM.Check(
                        ).checksum(
                            self.Pmets_objpath,
                            self.PMetsMessageDigestAlgorithm)  # Checksum
                        if errno:
                            self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
                                self.Cmets_objpath, str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug('Checksum for PMETS file: %s',
                                          self.Pmets_obj_checksum)
                            self.PMetsMessageDigestAlgorithm = 1
                            self.PMetsMessageDigest = self.Pmets_obj_checksum
                    ########################
                    # get checksum for CMETS file
                    if self.ok:
                        self.Cmets_obj_checksum, errno, why = ESSPGM.Check(
                        ).checksum(
                            self.Cmets_objpath,
                            self.CMetsMessageDigestAlgorithm)  # Checksum
                        if errno:
                            self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
                                self.Cmets_objpath, str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            logging.debug('Checksum for CMETS file: %s',
                                          self.Cmets_obj_checksum)
                    ###################################################
                    # Check CMetsMessageDigest
                    if self.ok:
                        if self.ValidateChecksum:
                            if self.CMetsMessageDigest == self.Cmets_obj_checksum:
                                logging.info(
                                    'Succeeded to verify Content Mets MessageDigest for object: %s',
                                    self.ObjectIdentifierValue)
                            else:
                                self.event_info = 'Failed to verify Content Mets MessageDigest for object: %s, CMetsMessageDigest: %s is equal to FileMessageDigest: %s' % (
                                    self.ObjectIdentifierValue,
                                    str(self.CMetsMessageDigest),
                                    str(self.Cmets_obj_checksum))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                        else:
                            logging.warning(
                                'Checksum validate is disabled for object: %s',
                                self.ObjectIdentifierValue)
                    if self.ValidateXML:
                        ########################
                        # XML Schema Validate CMETS file
                        if self.ok:
                            errno, why = ESSMD.validate(
                                FILENAME=self.Cmets_objpath)
                            if errno:
                                self.event_info = 'Failed to schema validate Content METS file for object: %s, why: %s' % (
                                    self.ObjectIdentifierValue, str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.info(
                                    'Succeeded to schema validate Content METS file for object: %s',
                                    self.ObjectIdentifierValue)
                        ########################
                        # XML Schema Validate PMETS file
                        if self.ok:
                            errno, why = ESSMD.validate(
                                FILENAME=self.Pmets_objpath)
                            if errno:
                                self.event_info = 'Failed to schema validate Package METS file for object: %s, why: %s' % (
                                    self.ObjectIdentifierValue, str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.info(
                                    'Succeeded to schema validate Package METS file for object: %s',
                                    self.ObjectIdentifierValue)
                        ########################
                        # XML Schema Validate PREMIS file
                        if self.ok and self.premis_obj:
                            errno, why = ESSMD.validate(
                                FILENAME=self.premis_objpath)
                            if errno:
                                self.event_info = 'Failed to schema validate PREMIS file for object: %s, why: %s' % (
                                    self.ObjectIdentifierValue, str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.info(
                                    'Succeeded to schema validate PREMIS file for object: %s',
                                    self.ObjectIdentifierValue)
                        ########################
                        # XML Schema Validate ADDML file
                        if self.ok and self.addml_obj:
                            errno, why = ESSMD.validate(
                                FILENAME=self.addml_objpath)
                            if errno:
                                self.event_info = 'Failed to schema validate ADDML file for object: %s, why: %s' % (
                                    self.ObjectIdentifierValue, str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1050', '', 'ESSArch AIPValidate',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                            else:
                                logging.info(
                                    'Succeeded to schema validate ADDML file for object: %s',
                                    self.ObjectIdentifierValue)
                    else:
                        logging.warning(
                            'Schema validate XML is disabled for object: %s',
                            self.ObjectIdentifierValue)
                    self.stopVerTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    self.VerTime = self.stopVerTime - self.startVerTime
                    if self.VerTime.seconds < 1:
                        self.VerTime = datetime.timedelta(
                            seconds=1
                        )  #Fix min time to 1 second if it is zero.
                    self.ObjectSizeMB = int(self.ObjectSize) / 1048576
                    self.VerMBperSEC = int(self.ObjectSizeMB) / int(
                        self.VerTime.seconds)
                    ##################################
                    # Write metadatafiles to DB-blob or FTP
                    if self.ok:
                        if ESSDB.DB().action('ESSConfig', 'GET', ('Value', ),
                                             ('Name', 'MD_FTP_HOST'))[0][0]:
                            if self.Cmets_obj:
                                ##################################
                                # Write CMETS metadatafile to DB-blob
                                res, errno, why = ESSmetablob.prod(
                                ).StoreMetadataBlob(
                                    ObjectUUID=self.ObjectUUID,
                                    ObjectIdentifierValue=self.
                                    ObjectIdentifierValue,
                                    ObjectMetadataType=26,
                                    FILENAME=self.Cmets_objpath,
                                    FTPFileName=string.replace(
                                        self.Cmets_obj,
                                        self.ObjectIdentifierValue + '/', ''),
                                    FTPflag=1,
                                    DBflag=0,
                                    AgentIdentifierValue=AgentIdentifierValue)
                                if errno:
                                    self.event_info = 'Failed to store Content METS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
                                        self.ObjectIdentifierValue, str(errno),
                                        str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1050', '', 'ESSArch AIPValidate',
                                        ProcVersion, '1', self.event_info, 2,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info(
                                        'Succeeded to store Content METS file to FTP server or DB-blob for object: %s',
                                        self.ObjectIdentifierValue)
                            if self.premis_obj:
                                ##################################
                                # Write PREMIS metadatafile to DB-blob
                                res, errno, why = ESSmetablob.prod(
                                ).StoreMetadataBlob(
                                    ObjectUUID=self.ObjectUUID,
                                    ObjectIdentifierValue=self.
                                    ObjectIdentifierValue,
                                    ObjectMetadataType=27,
                                    FILENAME=self.premis_objpath,
                                    FTPFileName=self.premis_obj,
                                    FTPflag=1,
                                    DBflag=0,
                                    AgentIdentifierValue=AgentIdentifierValue)
                                if errno:
                                    self.event_info = 'Failed to store PREMIS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
                                        self.ObjectIdentifierValue, str(errno),
                                        str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1050', '', 'ESSArch AIPValidate',
                                        ProcVersion, '1', self.event_info, 2,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info(
                                        'Succeeded to store PREMIS file to FTP server or DB-blob for object: %s',
                                        self.ObjectIdentifierValue)
                            if self.addml_obj:
                                ##################################
                                # Write ADDML metadatafile to DB-blob
                                res, errno, why = ESSmetablob.prod(
                                ).StoreMetadataBlob(
                                    ObjectUUID=self.ObjectUUID,
                                    ObjectIdentifierValue=self.
                                    ObjectIdentifierValue,
                                    ObjectMetadataType=25,
                                    FILENAME=self.addml_objpath,
                                    FTPFileName=self.addml_obj,
                                    FTPflag=1,
                                    DBflag=0,
                                    AgentIdentifierValue=AgentIdentifierValue)
                                if errno:
                                    self.event_info = 'Failed to store ADDML file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
                                        self.ObjectIdentifierValue, str(errno),
                                        str(why))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1050', '', 'ESSArch AIPValidate',
                                        ProcVersion, '1', self.event_info, 2,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                else:
                                    logging.info(
                                        'Succeeded to store ADDML file to FTP server or DB-blob for object: %s',
                                        self.ObjectIdentifierValue)
                        else:
                            logging.info(
                                'Skip to store metadata to FTP server or DB-blob for object: %s',
                                self.ObjectIdentifierValue)
                    if not self.ok:
                        errno, why = ESSPGM.DB().SetAIPstatus(
                            self.IngestTable, self.ext_IngestTable,
                            AgentIdentifierValue, self.ObjectUUID, 51, 4)
                        if errno:
                            logging.error(
                                'Failed to update DB status for AIP: ' +
                                str(self.ObjectIdentifierValue) + ' error: ' +
                                str(why))
                        else:
                            self.event_info = 'Failed to validate AIP package: ' + self.ObjectIdentifierValue
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                    else:
                        logging.info('Succeeded to validate AIP package: ' +
                                     self.ObjectIdentifierValue + ' , ' +
                                     str(self.VerMBperSEC) +
                                     ' MB/Sec and Time: ' + str(self.VerTime))
                        self.timestamp_utc = datetime.datetime.utcnow(
                        ).replace(microsecond=0, tzinfo=pytz.utc)
                        self.timestamp_dst = self.timestamp_utc.astimezone(
                            self.tz)
                        res, errno, why = ESSDB.DB().action(
                            self.IngestTable, 'UPD',
                            ('StatusProcess', '59', 'StatusActivity', '0',
                             'CMetaMessageDigestAlgorithm',
                             self.CMetsMessageDigestAlgorithm,
                             'CMetaMessageDigest', self.CMetsMessageDigest,
                             'PMetaMessageDigestAlgorithm',
                             self.PMetsMessageDigestAlgorithm,
                             'PMetaMessageDigest', self.PMetsMessageDigest,
                             'LastEventDate',
                             self.timestamp_utc.replace(tzinfo=None),
                             'linkingAgentIdentifierValue',
                             AgentIdentifierValue, 'LocalDBdatetime',
                             self.timestamp_utc.replace(tzinfo=None)),
                            ('ObjectIdentifierValue',
                             self.ObjectIdentifierValue))
                        if errno:
                            logging.error('Failed to update Local DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(why))
                        else:
                            ESSPGM.Events().create('1050', '',
                                                   'ESSArch AIPValidate',
                                                   ProcVersion, '0', '', 2,
                                                   self.ObjectIdentifierValue)
                        if errno == 0 and ExtDBupdate:
                            ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
                                self.IngestTable, 'UPD',
                                ('StatusProcess', '59', 'StatusActivity', '0',
                                 'CMetaMessageDigestAlgorithm',
                                 self.CMetsMessageDigestAlgorithm,
                                 'CMetaMessageDigest', self.CMetsMessageDigest,
                                 'PMetaMessageDigestAlgorithm',
                                 self.PMetsMessageDigestAlgorithm,
                                 'PMetaMessageDigest', self.PMetsMessageDigest,
                                 'LastEventDate',
                                 self.timestamp_dst.replace(tzinfo=None),
                                 'linkingAgentIdentifierValue',
                                 AgentIdentifierValue),
                                ('ObjectIdentifierValue',
                                 self.ObjectIdentifierValue))
                            if ext_errno:
                                logging.error(
                                    'Failed to update External DB: ' +
                                    str(self.ObjectIdentifierValue) +
                                    ' error: ' + str(ext_why))
                            else:
                                res, errno, why = ESSDB.DB().action(
                                    self.IngestTable, 'UPD',
                                    ('ExtDBdatetime',
                                     self.timestamp_utc.replace(tzinfo=None)),
                                    ('ObjectIdentifierValue',
                                     self.ObjectIdentifierValue))
                                if errno:
                                    logging.error(
                                        'Failed to update Local DB: ' +
                                        str(self.ObjectIdentifierValue) +
                                        ' error: ' + str(why))

                elif self.metatype == 0:  #self.metatype 0 = No metadata
                    logging.info('Skip to validate AIP package: ' +
                                 self.ObjectIdentifierValue)
                    self.timestamp_utc = datetime.datetime.utcnow().replace(
                        microsecond=0, tzinfo=pytz.utc)
                    self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                    res, errno, why = ESSDB.DB().action(
                        self.IngestTable, 'UPD',
                        ('StatusProcess', '59', 'StatusActivity', '0',
                         'CMetaMessageDigestAlgorithm', '0',
                         'PMetaMessageDigestAlgorithm', '0', 'LastEventDate',
                         self.timestamp_utc.replace(tzinfo=None),
                         'linkingAgentIdentifierValue', AgentIdentifierValue,
                         'LocalDBdatetime',
                         self.timestamp_utc.replace(tzinfo=None)),
                        ('ObjectIdentifierValue', self.ObjectIdentifierValue))
                    if errno:
                        logging.error('Failed to update Local DB: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    else:
                        ESSPGM.Events().create('1050', '',
                                               'ESSArch AIPValidate',
                                               ProcVersion, '0',
                                               'Skip to validate AIP package',
                                               2, self.ObjectIdentifierValue)
                    if errno == 0 and ExtDBupdate:
                        ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
                            self.IngestTable, 'UPD',
                            ('StatusProcess', '59', 'StatusActivity', '0',
                             'CMetaMessageDigestAlgorithm', '0',
                             'PMetaMessageDigestAlgorithm', '0',
                             'LastEventDate',
                             self.timestamp_dst.replace(tzinfo=None),
                             'linkingAgentIdentifierValue',
                             AgentIdentifierValue),
                            ('ObjectIdentifierValue',
                             self.ObjectIdentifierValue))
                        if ext_errno:
                            logging.error('Failed to update External DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(ext_why))
                        else:
                            res, errno, why = ESSDB.DB().action(
                                self.IngestTable, 'UPD',
                                ('ExtDBdatetime',
                                 self.timestamp_utc.replace(tzinfo=None)),
                                ('ObjectIdentifierValue',
                                 self.ObjectIdentifierValue))
                            if errno:
                                logging.error('Failed to update Local DB: ' +
                                              str(self.ObjectIdentifierValue) +
                                              ' error: ' + str(why))
            db.close_old_connections()
            self.mLock.release()
            time.sleep(int(self.Time))
        self.mDieFlag = 0
    def ThreadMain(self,ProcName):
        logging.info('Starting ' + ProcName)
        while 1:
            if self.mDieFlag==1: break      # Request for death
            self.mLock.acquire()
            self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
            if self.Run == '0':
                logging.info('Stopping ' + ProcName)
                ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                self.RunFlag=0
                self.mLock.release()
                #if Debug: print 'RunFlag: 0'
                time.sleep(2)
                continue
            # Process Item 
            lock=thread.allocate_lock()
            self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
            self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
            if ExtDBupdate:
                self.ext_IngestTable = self.IngestTable
            else:
                self.ext_IngestTable = ''
            #if Debug: logging.info('Start to list worklist (self.dbget)')

            self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue','ObjectUUID','PolicyId','INFORMATIONCLASS'),
                                                                             ('StatusActivity','=','0','AND',
                                                                              'StatusProcess','BETWEEN',24,'AND',26))
            if errno: logging.error('Failed to access Local DB, error: ' + str(why))
            for self.obj in self.dbget:
                self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                if self.ProcDB[0]=='0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                    thread.interrupt_main()
                    time.sleep(5)
                    break
                elif self.ProcDB[1]==1:
                    while 1:
                        time.sleep(60)
                        self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                        if self.ProcDB[1]==1:
                            logging.info('Process is in pause state')
                        else:
                            break
                self.ok = 1
                ###########################################################
                # get policy info
                self.ObjectIdentifierValue = ESSPGM.Check().str2unicode(self.obj[0])
                self.ObjectUUID = self.obj[1]
                self.PolicyId = self.obj[2]
                self.INFORMATIONCLASS = self.obj[3]
                logging.info('Start to validate format for SIP: %s', self.ObjectIdentifierValue)
                self.ChecksumAlgorithm_CHOICES_dict = dict(ChecksumAlgorithm_CHOICES)
                self.ChecksumAlgorithm_CHOICES_invdict = ESSPGM.Check().invert_dict(self.ChecksumAlgorithm_CHOICES_dict)
                self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','INFORMATIONCLASS','ChecksumAlgorithm','IngestPath','Mode'),('PolicyID',self.PolicyId))
                if errno: 
                    logging.error('Failed to access Local DB, error: ' + str(why)) 
                    self.ok = 0
                if self.ok:
                    ###########################################################
                    # set variables
                    self.AIPpath = ESSPGM.Check().str2unicode(self.PolicyDB[0][0])
                    self.metatype = self.PolicyDB[0][1]
                    self.Policy_INFORMATIONCLASS = self.PolicyDB[0][2]
                    self.ChecksumAlgorithm = self.PolicyDB[0][3]
                    self.ChecksumAlgorithm_name = self.ChecksumAlgorithm_CHOICES_dict[self.ChecksumAlgorithm]
                    self.SIPpath = ESSPGM.Check().str2unicode(self.PolicyDB[0][4])
                    self.DBmode = self.PolicyDB[0][5]
                    logging.debug('self.obj: %s', str(self.obj))
                    logging.debug('self.ObjectIdentifierValue: %s', self.ObjectIdentifierValue)
                    logging.debug('Len self.ObjectIdentifierValue: %s', len(self.ObjectIdentifierValue))
                    logging.debug('self.SIPpath: %s', self.SIPpath)
                    logging.debug('self.AIPpath: %s', self.AIPpath)
                if self.metatype == 2:
                    ############################################
                    # Create PREMISfile from TIFFEdit.RES if metatype is 2
                    logging.info('Start to convert RESfile to PREMISfile for object: ' + self.ObjectIdentifierValue)
                    self.xml_PREMIS,self.errno,self.why = ESSMD.RES2PREMIS(os.path.join(self.SIPpath,self.ObjectIdentifierValue),AgentIdentifierValue[8:])
                    if self.errno == 10:
                        self.event_info = 'Failed to parse RESfile, error.num: %s error.det: %s' % (str(self.errno),str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                    elif self.errno == 20:
                        self.event_info = 'I/O error to access RESfile, error.num: %s error.det: %s' % (str(self.errno),str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                    elif self.errno == 30:
                        self.event_info = 'Validation errors for PREMIS file, error.num: %s error.det: %s' % (str(self.errno),str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                    elif self.errno == 40:
                        self.event_info = 'Problem to write PREMIS file, error.num: %s error.det: %s' % (str(self.errno),str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                    if self.errno > 1:
                        self.event_info = 'Problem to convert RES to PREMIS for SIP package: %s, error.num: %s  error.desc: %s' % (self.ObjectIdentifierValue,str(self.errno),str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                        self.ok = 0
                    elif self.errno == 1:
                        self.event_info = 'Warning in convert RES to PREMIS for SIP package: %s, error.num: %s  warning.desc: %s' % (self.ObjectIdentifierValue,str(self.errno),str(self.why))
                        logging.warning(self.event_info)
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'0',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                    else:
                        ESSPGM.Events().create('1022','RES2PREMIS','ESSArch SIPValidateFormat',ProcVersion,'0','',self.DBmode,self.ObjectIdentifierValue)
                    if self.ok:
                        ############################################
                        # Clean RES SIP from "junk" files 
                        self.errno,self.why = ESSPGM.Check().CleanRES_SIP(os.path.join(self.SIPpath,self.ObjectIdentifierValue))
                        if self.errno:
                            self.event_info = 'Problem to clean RES SIP from "junk files" for SIP package: %s, error.num: %s  error.desc: %s' % (self.ObjectIdentifierValue,str(self.errno),str(self.why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1022','CleanRES_SIP','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            ESSPGM.Events().create('1022','CleanRES_SIP','ESSArch SIPValidateFormat',ProcVersion,'0','',self.DBmode,self.ObjectIdentifierValue)
                elif self.metatype == 1:
                    ###########################################################
                    # Create PREMISfile from Content_METS if metatype is 1
                    res,errno,why = ESSMD.METS2PREMIS(self.SIPpath,self.ObjectIdentifierValue)
                    if not errno:
                        logging.info('Succeeded to convert Content_METS to PREMISfile for information package: %s', self.ObjectIdentifierValue)
                    else:
                        self.event_info = 'Problem to convert Content_METS to PREMISfile for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                        logging.error(self.event_info)
                        #ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                        self.ok = 0
                elif self.metatype in [4]:
                    self.SIPpath = os.path.join(self.SIPpath,self.ObjectIdentifierValue)
                if self.ok:
                    if self.metatype in [1,2,3]:
                        ###########################################################
                        # get object_list from PREMIS file
                        self.Premis_filepath = u'%s/%s/%s_PREMIS.xml' % (self.SIPpath,self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        self.object_list,errno,why = ESSMD.getPremisObjects(FILENAME=self.Premis_filepath)
                        if errno == 0:
                            logging.info('Succeeded to get object_list from premis for information package: %s', self.ObjectIdentifierValue)
                        else:
                            self.event_info = 'Problem to get object_list from premis for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                            self.ok = 0
                    elif self.metatype in [4]:
                        ###########################################################
                        # get object_list from METS
                        if os.path.exists(os.path.join(self.SIPpath,'sip.xml')):
                            mets_file = 'sip.xml'
                            self.SIPmets_objpath = os.path.join(self.SIPpath,mets_file)
                        elif os.path.exists(os.path.join(self.SIPpath,'mets.xml')):
                            mets_file = 'mets.xml'
                            self.SIPmets_objpath = os.path.join(self.SIPpath,mets_file)
                        #elif os.path.exists(os.path.join(self.SIPpath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
                        #    mets_file = '%s_Content_METS.xml' % self.ObjectIdentifierValue
                        #    self.SIPmets_objpath = os.path.join(self.SIPpath,mets_file)
                        else:
                            self.SIPmets_objpath = ''
                        self.object_list,errno,why = ESSMD.getAIPObjects(FILENAME=self.SIPmets_objpath)
                        if errno == 0:
                            logging.info('Succeeded to get object_list from METS for information package: %s', self.ObjectIdentifierValue)
                            self.F_Checksum,errno,why = ESSPGM.Check().checksum(self.SIPmets_objpath, self.ChecksumAlgorithm) # Checksum
                            self.F_SIZE = os.stat(self.SIPmets_objpath)[6]
                            self.object_list.append([mets_file,self.ChecksumAlgorithm_name,self.F_Checksum,self.F_SIZE,''])
                        else:
                            self.event_info = 'Problem to get object_list from METS for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                            self.ok = 0
                if self.ok:
                    ###########################################################
                    # update ObjectIdentifierValue to StatusProcess: 25 and StatusActivity: 5
                    errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 25, 5)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                    logging.info('Format validate object: ' + self.ObjectIdentifierValue)
                if self.ok:
                    ###########################################################
                    # Start to format validate SIP
                    self.startTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                    self.ObjectNumItems = 0
                    self.ObjectSize = 0
                    if self.metatype == 1:
                        ############################################
                        # Object have metatype 1 (METS)
                        self.tmp_object_id = (u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETAxmlWrap', 'PREMIS'])
                    elif self.metatype == 2:
                        ############################################
                        # Object have metatype 2 (RES)
                        #self.tmp_object_id = ('%s/TIFFEdit.RES') % self.ObjectIdentifierValue
                        #self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETA', ''])
                        self.tmp_object_id = (u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETAxmlWrap', 'PREMIS'])
                    elif self.metatype == 3:
                        ############################################
                        # Object have metatype 3 (ADDML)
                        self.tmp_object_id = (u'%s/%s_ADDML.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETAxmlWrap', 'ADDML'])
                        self.tmp_object_id = (u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETAxmlWrap', 'PREMIS'])
                    for self.object in self.object_list:
                        logging.debug('variable self.SIPpath: %s, type: %s' % (self.SIPpath,type(self.SIPpath)))
                        logging.debug('variable self.object[0]: %s, type: %s' % (self.object[0],type(self.object[0])))
                        self.filepath = os.path.join(self.SIPpath, self.object[0])
                        logging.debug('variable self.filepath: %s, type: %s' % (self.filepath,type(self.filepath)))
                        #self.filepath = ESSPGM.Check().Unicode2isoStr(self.filepath.encode('utf-8'))
                        #self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
                        #logging.debug('variable self.filepath_iso: %s, type: %s' % (self.filepath_iso,type(self.filepath_iso)))
                        if self.metatype in [1,2,3] and self.ObjectNumItems == 0:
                            if self.object[0] == self.ObjectIdentifierValue:
                                logging.info('First premis object match information package: %s', self.ObjectIdentifierValue)
                            else:
                                self.event_info = 'First premis object do not match information package: %s, premis_object: %s' % (self.ObjectIdentifierValue,self.object[0])
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                self.ok = 0
                                break
                            if self.ok and os.access(self.filepath,os.X_OK):
                                pass
                            else:
                                self.event_info = 'Object path: %s do not exist or is not executable!' % self.filepath
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                self.ok = 0
                                break
                        if self.ok and os.access(self.filepath,os.R_OK):
                            pass
                        else:
                            self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                            self.ok = 0
                            break
                        if self.ok and os.access(self.filepath,os.W_OK):
                            pass
                        else:
                            self.event_info = 'Missing permission, Object path: %s is not writeable!' % self.filepath
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                            self.ok = 0
                            break
                        if self.metatype in [1,2,3]:
                            if self.ok and not (self.ObjectNumItems == 0 or self.object[5] == 'ARCHMETA' or self.object[5] == 'ARCHMETAxmlWrap'):
                                if int(os.stat(self.filepath)[6]) == int(self.object[4]):
                                    self.ObjectSize += int(self.object[4])
                                else:
                                    self.event_info = 'Filesize for object path: %s is %s and premis object size is %s. The sizes must match!' % (self.filepath,str(os.stat(self.filepath)[6]),str(self.object[4]))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                                if self.ok:
                                    self.F_Checksum,errno,why = ESSPGM.Check().checksum(self.filepath,self.object[1]) # Checksum
                                    if errno:
                                        self.event_info = 'Failed to get checksum for: %s, Error: %s' % (self.filepath,str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                        self.ok = 0
                                if self.ok:
                                    if self.F_Checksum == self.object[2]:
                                        pass
                                    else:
                                        self.event_info = 'Checksum for object path: %s is %s and premis object checksum is %s. The checksum must match!' % (self.filepath,self.F_Checksum,self.object[2])
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                        self.ok = 0
                                        break
                            elif self.ok and not self.ObjectNumItems == 0 and (self.object[5] == 'ARCHMETA' or self.object[5] == 'ARCHMETAxmlWrap'):
                                if int(os.stat(self.filepath)[6]) > 0:
                                    pass
                                else:
                                    self.event_info = 'Filesize for object path: %s is 0 bytes. The size should be more then 0 bytes!' % self.filepath
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                        elif self.metatype in [4]:
                            if self.ok:
                                #[objectIdentifierValue,messageDigestAlgorithm,messageDigest,a_SIZE,a_MIMETYPE]
                                if int(os.stat(self.filepath)[6]) == int(self.object[3]):
                                    self.ObjectSize += int(self.object[3])
                                else:
                                    self.event_info = 'Filesize for object path: %s is %s and METS object size is %s. The sizes must match!' % (self.filepath,str(os.stat(self.filepath)[6]),str(self.object[3]))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                                if self.ok:
                                    self.F_Checksum,errno,why = ESSPGM.Check().checksum(self.filepath,self.object[1]) # Checksum
                                    if errno:
                                        self.event_info = 'Failed to get checksum for: %s, Error: %s' % (self.filepath,str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                        self.ok = 0
                                if self.ok:
                                    if self.F_Checksum == self.object[2]:
                                        pass
                                    else:
                                        self.event_info = 'Checksum for object path: %s is %s and METS object checksum is %s. The checksum must match!' % (self.filepath,self.F_Checksum,self.object[2])
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                        self.ok = 0
                                        break
                        self.ObjectNumItems += 1
                if self.ok:
                    if self.metatype in [1,2,3]:
                        ###############################################################################
                        # Check if SIP filesystem path contain files that not exist in metadatafile
                        for self.filesystem_object in ESSPGM.Check().GetFiletree(os.path.join(self.SIPpath,self.ObjectIdentifierValue)):
                            self.missmatch_flag = 0
                            for self.object in self.object_list:
                                #if os.path.join(self.ObjectIdentifierValue,self.filesystem_object) == self.object[0].encode('utf-8'):
                                if os.path.join(self.ObjectIdentifierValue,self.filesystem_object) == self.object[0]:
                                    self.missmatch_flag = 0
                                    break
                                else:
                                    self.missmatch_flag = 1
                            if self.missmatch_flag:
                                self.filesystempath = u'%s/%s/%s' % (self.SIPpath,self.ObjectIdentifierValue,self.filesystem_object)
                                self.event_info = 'Filesystem file: %s do not exist in metadatafile for object: %s' % (self.filesystempath,os.path.join(self.SIPpath,self.ObjectIdentifierValue))
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                self.ok = 0
                    if self.metatype in [4]:
                        ###############################################################################
                        # Check if SIP filesystem path contain files that not exist in metadatafile
                        for self.filesystem_object in ESSPGM.Check().GetFiletree(self.SIPpath):
                            self.missmatch_flag = 0
                            for self.object in self.object_list:
                                if self.filesystem_object == self.object[0]:
                                    self.missmatch_flag = 0
                                    break
                                else:
                                    self.missmatch_flag = 1
                            if self.missmatch_flag:
                                self.filesystempath = u'%s/%s' % (self.SIPpath,self.filesystem_object)
                                self.event_info = 'Filesystem file: %s do not exist in metadatafile for object: %s' % (self.filesystempath,self.SIPpath)
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                                self.ok = 0
                if self.ok:
                    if self.metatype in [4]:
                        ###############################################################################
                        # Check if SIP INFORMATIONCLASS match Policy
                        if self.INFORMATIONCLASS == self.Policy_INFORMATIONCLASS:
                            self.event_info = 'Object: %s InformationClass: %s match defined InformaionClass: %s in PolicyID: %s' % (self.ObjectIdentifierValue,self.INFORMATIONCLASS,self.Policy_INFORMATIONCLASS,self.PolicyId)
                            logging.info(self.event_info)
                        else:
                            self.event_info = 'Object: %s InformationClass: %s do not match defined InformationClass: %s in PolicyID: %s' % (self.ObjectIdentifierValue,self.INFORMATIONCLASS,self.Policy_INFORMATIONCLASS,self.PolicyId)
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                            self.ok = 0
                if self.ok:
                    self.stopTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                    self.MeasureTime = self.stopTime-self.startTime
                    self.ObjectSizeMB = self.ObjectSize/1048576 
                    if self.MeasureTime.seconds < 1: self.MeasureTime = datetime.timedelta(seconds=1)	#Fix min time to 1 second if it is zero.
                    self.VerMBperSEC = int(self.ObjectSizeMB)/int(self.MeasureTime.seconds)
                if self.ok:
                    logging.info('Succeeded to validate SIP package: ' + self.ObjectIdentifierValue + ' , ' + str(self.VerMBperSEC) + ' MB/Sec and Time: ' + str(self.MeasureTime))
                    errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 29, 0)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                    else: 
                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'0','',self.DBmode,self.ObjectIdentifierValue)
                else:
                    errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 26, 4)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                    else:
                        self.event_info = 'Failed to validate SIP package: ' + self.ObjectIdentifierValue
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
            db.close_old_connections()
            self.mLock.release()
            time.sleep(int(self.Time))
        self.mDieFlag=0
Exemple #7
0
    def ThreadMain(self,ProcName):
        logging.info('Starting ' + ProcName)
        TimeZone = timezone.get_default_timezone_name()
        self.tz=pytz.timezone(TimeZone)
        METS_NAMESPACE = SchemaProfile.objects.get(entity = 'mets_namespace').value
        METS_SCHEMALOCATION = SchemaProfile.objects.get(entity = 'mets_schemalocation').value
        METS_PROFILE = SchemaProfile.objects.get(entity = 'mets_profile').value
        XLINK_NAMESPACE = SchemaProfile.objects.get(entity = 'xlink_namespace').value
        XSI_NAMESPACE = SchemaProfile.objects.get(entity = 'xsi_namespace').value
        while 1:
                if self.mDieFlag==1: break      # Request for death
                self.mLock.acquire()
                self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
                if self.Run == '0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                    self.RunFlag=0
                    self.mLock.release()
                    if Debug: logging.info('RunFlag: 0')
                    time.sleep(2)
                    continue
                # Process Item 
                lock=thread.allocate_lock()
                Cmets_obj = Parameter.objects.get(entity='content_descriptionfile').value
                self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
                self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
                if ExtDBupdate:
                    self.ext_IngestTable = self.IngestTable
                else:
                    self.ext_IngestTable = ''
                self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue',
                                                                                  'ObjectUUID',
                                                                                  'PolicyId',
                                                                                  'ObjectSize'),
                                                                                 ('StatusProcess','BETWEEN',39,'AND',40,'AND',
                                                                                  'StatusActivity','=','0'))
                if errno: logging.error('Failed to access Local DB, error: ' + str(why))
                for self.obj in self.dbget:
                    self.ok = 1
                    self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                    if self.ProcDB[0]=='0':
                        logging.info('Stopping ' + ProcName)
                        ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                        thread.interrupt_main()
                        time.sleep(5)
                        break
                    elif self.ProcDB[1]==1:
                        while 1:
                            time.sleep(60)
                            self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                            if self.ProcDB[1]==1:
                                logging.info('Process is in pause state')
                            else:
                                break
                    self.ObjectIdentifierValue = self.obj[0]
                    self.ObjectUUID = self.obj[1]
                    self.PolicyId = self.obj[2]
                    self.ObjectSize = self.obj[3]
                    self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','ChecksumAlgorithm','IngestPath'),('PolicyID',self.PolicyId))
                    if errno:
                        logging.error('Failed to access Local DB, error: ' + str(why))
                        self.ok = 0
                    if self.ok:
                        ###########################################################
                        # set variables
                        self.AIPpath = self.PolicyDB[0][0]
                        self.metatype = self.PolicyDB[0][1]
                        self.ChecksumAlgorithm = self.PolicyDB[0][2]
                        self.CA = dict(ChecksumAlgorithm_CHOICES)[self.ChecksumAlgorithm]
                        self.SIPpath = self.PolicyDB[0][3]
                        self.p_obj = self.ObjectIdentifierValue + '.tar'
                        self.ObjectPath = os.path.join(self.AIPpath,self.p_obj)
                        self.SIProotpath = os.path.join(self.SIPpath,self.ObjectIdentifierValue)
                        if self.metatype in [4]:
                            #self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                            #self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
                            #self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
                            self.Cmets_obj = Cmets_obj.replace('{objid}',self.ObjectIdentifierValue)
                            self.Cmets_objpath = os.path.join(self.SIProotpath,self.Cmets_obj)
                        elif self.metatype in [1,2,3]:
                            self.Cmets_obj = '%s_Content_METS.xml' % (self.ObjectIdentifierValue)
                            self.Cmets_objpath = os.path.join(self.AIPpath,self.Cmets_obj)
                        self.Pmets_obj = '%s_Package_METS.xml' % (self.ObjectIdentifierValue)
                        self.Pmets_objpath = os.path.join(self.AIPpath,self.Pmets_obj)
                        self.AIC_UUID = None
                        self.AIC_UUID_rel_ObjectUUIDs = []
                    if self.ok:
                        METS_agent_list = []
                        METS_altRecordID_list = []
                        if self.metatype == 1:
                            ############################################
                            # Object have metatype 1 (METS)
                            self.METS_LABEL = 'ESSArch AIP'
                            # Get SIP Content METS information
                            self.METSfilepath = os.path.join(self.SIPpath,self.ObjectIdentifierValue + '/metadata/SIP/' + self.ObjectIdentifierValue + '_Content_METS.xml')
                            res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.METSfilepath)
                            for agent in res_info[2]:
                                if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
                                    METS_agent_list.append(agent)
                            METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                            METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                        elif self.metatype == 2:
                            ############################################
                            # Object have metatype 2 (RES)
                            self.METS_LABEL = 'Imaging AIP RA'
                            METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
                            METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
                            METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                            METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                        elif self.metatype == 3:
                            ############################################
                            # Object have metatype 3 (ADDML)
                            self.METS_LABEL = 'Born Digital AIP RA'
                            METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
                            METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
                            METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                            METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                        elif self.metatype in [4]:
                            ############################################
                            # Object have metatype 4 (eARD METS)
                            res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.Cmets_objpath)
                            for agent in res_info[2]:
                                #if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
                                    METS_agent_list.append(agent)
                            self.METS_LABEL = res_info[0][0]
                            METS_agent_list.append(['CREATOR',None, 'INDIVIDUAL',None,AgentIdentifierValue,[]])
                            METS_agent_list.append(['CREATOR',None, 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                            for altRecordID in res_info[3]:
                                METS_altRecordID_list.append(altRecordID)
                    logging.debug('self.obj: '+str(self.obj))
                    if self.ChecksumAlgorithm > 0: #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
                        self.startCalTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                        errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 40, 5)
                        if errno: logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                        logging.info('Start create Package METS for: ' + self.ObjectIdentifierValue)
                        if self.ok:
                            ###########################################################
                            # Create PMETS for AIP package
                            self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(self.Cmets_objpath,self.CA)
                            if errno:
                                self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(self.Cmets_objpath)
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            self.M_statinfo = os.stat(self.Cmets_objpath)
                            self.M_SIZE = self.M_statinfo.st_size
                            self.M_utc_mtime = datetime.datetime.utcfromtimestamp(self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                            self.M_lociso_mtime = self.M_utc_mtime.astimezone(self.tz).isoformat()
                            self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(self.ObjectPath,self.CA)
                            if errno:
                                self.event_info = 'Problem to get checksum for AIP package: ' + str(self.ObjectPath)
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                self.ok = 0
                            self.P_statinfo = os.stat(self.ObjectPath)
                            self.P_SIZE = self.P_statinfo.st_size
                            self.P_utc_mtime = datetime.datetime.utcfromtimestamp(self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                            self.P_lociso_mtime = self.P_utc_mtime.astimezone(self.tz).isoformat()
        
                            if self.metatype in [1,2,3]:
                                self.PMETSdoc = ESSMD.createPMets(
                                    ID=self.ObjectIdentifierValue,
                                    LABEL=self.METS_LABEL,
                                    AGENT=METS_agent_list,
                                    P_SIZE=self.P_SIZE,
                                    P_CREATED=self.P_lociso_mtime,
                                    P_CHECKSUM=self.P_CHECKSUM,
                                    P_CHECKSUMTYPE=self.CA,
                                    M_SIZE=self.M_SIZE,
                                    M_CREATED=self.M_lociso_mtime,
                                    M_CHECKSUM=self.M_CHECKSUM,
                                    M_CHECKSUMTYPE=self.CA,
                                )
                                errno,why = ESSMD.writeToFile(self.PMETSdoc,self.Pmets_objpath)
                                if errno:
                                    self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(self.Pmets_objpath)
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                    time.sleep(2)
                                    self.ok = 0
                            elif self.metatype in [4]:
                                ms_files = []
                                ms_files.append(['amdSec', None, 'techMD', 'techMD001', None,
                                                 None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s/%s' % (self.ObjectIdentifierValue,self.Cmets_obj), 'simple',
                                                 self.M_CHECKSUM, self.CA, self.M_SIZE, 'text/xml', self.M_lociso_mtime,
                                                 'OTHER', 'METS', None])

                                ms_files.append(['fileSec', None, None, None, None,
                                                 None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s' % self.p_obj, 'simple',
                                                 self.P_CHECKSUM, self.CA, self.P_SIZE, 'application/x-tar', self.P_lociso_mtime,
                                                 'tar', 'techMD001', None])                          
                                # define namespaces
                                self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
                                self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
                                self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
                                self.namespacedef += ' xsi:schemaLocation="%s %s"' % (METS_NAMESPACE, METS_SCHEMALOCATION)
                                                            
                                errno,info_list = ESSMD.Create_IP_mets(ObjectIdentifierValue = self.ObjectIdentifierValue, 
                                                                       METS_ObjectPath = self.Pmets_objpath,
                                                                       agent_list = METS_agent_list, 
                                                                       altRecordID_list = METS_altRecordID_list, 
                                                                       file_list = ms_files, 
                                                                       namespacedef = self.namespacedef, 
                                                                       METS_LABEL = self.METS_LABEL, 
                                                                       METS_PROFILE = METS_PROFILE, 
                                                                       METS_TYPE = 'AIP',  
                                                                       METS_DocumentID = self.Pmets_obj,
                                                                       TimeZone = TimeZone)
                                if errno:
                                    logging.error('Problem to create Package METS file, why: %s' % str(info_list))   

                        self.ObjectMessageDigest = self.P_CHECKSUM
                        self.stopCalTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                        self.CalTime = self.stopCalTime-self.startCalTime
                        self.ObjectSizeMB = self.ObjectSize/1048576
                        if self.CalTime.seconds < 1: self.CalTime = datetime.timedelta(seconds=1)   #Fix min time to 1 second if it is zero.
                        self.CalMBperSEC = int(self.ObjectSizeMB)/int(self.CalTime.seconds)
                        logging.info('Finished calculate checksum: ' + self.ObjectIdentifierValue + ' , ' + str(self.CalMBperSEC) + ' MB/Sec and Time: ' + str(self.CalTime))

                        if self.ok:
                            self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                            self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                            res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
                                                                                      'ObjectMessageDigest',self.ObjectMessageDigest,
                                                                                      'MetaObjectSize',self.M_SIZE,
                                                                                      'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                                      'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                                      'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                                     ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                            if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                            if errno == 0 and ExtDBupdate:
                                ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
                                                                                                         'ObjectMessageDigest',self.ObjectMessageDigest,
                                                                                                         'MetaObjectSize',self.M_SIZE,
                                                                                                         'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                                         'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                                        ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                                else:
                                    res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                    if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))

                        if self.ok and self.metatype == 4:
                            ####################################################
                            # Create AIC METS File:
                            aic_obj = ArchiveObject.objects.filter(relaic_set__UUID=self.ObjectUUID)[:1]
                            if aic_obj:
                                self.AIC_UUID = aic_obj.get().ObjectUUID
                                logging.info('Succeeded to get AIC_UUID: %s from DB' % self.AIC_UUID)
                            else:
                                logging.warning('AIC not found for IP object: %s, skip to create AIC METS file' % self.ObjectUUID)
                        if self.ok and self.AIC_UUID:
                            ip_obj_list = ArchiveObject.objects.filter(Q(StatusProcess=3000) | Q(ObjectUUID=self.ObjectUUID), reluuid_set__AIC_UUID=self.AIC_UUID).order_by('Generation')
                            if ip_obj_list:
                                logging.info('Start create AIC METS: ' + self.AIC_UUID)
                                self.AICmets_objpath = os.path.join(self.AIPpath,self.AIC_UUID + '_AIC_METS.xml')
                                ms_files = []
                                for ip_obj in ip_obj_list:
                                    logging.info('Add IP: %s to AIC METS: %s' % (ip_obj.ObjectUUID,self.AIC_UUID))
                                    ms_files.append(['fileSec', None, None, None, None,
                                          None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s' % ip_obj.ObjectUUID, 'simple',
                                          ip_obj.ObjectMessageDigest, dict(ChecksumAlgorithm_CHOICES)[ip_obj.ObjectMessageDigestAlgorithm], ip_obj.ObjectSize, 'application/x-tar', ip_obj.CreateDate,
                                          'IP Package', None, None])

                                # define namespaces
                                self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
                                self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
                                self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
                                self.namespacedef += ' xsi:schemaLocation="%s %s"' % (METS_NAMESPACE, METS_SCHEMALOCATION)

                                errno,info_list = ESSMD.Create_IP_mets(ObjectIdentifierValue = self.AIC_UUID, 
                                                                       METS_ObjectPath = self.AICmets_objpath,
                                                                       agent_list = [], 
                                                                       altRecordID_list = [], 
                                                                       file_list = ms_files, 
                                                                       namespacedef = self.namespacedef, 
                                                                       METS_LABEL = 'AIC relation to IP', 
                                                                       METS_PROFILE = METS_PROFILE, 
                                                                       METS_TYPE = 'AIC',  
                                                                       METS_DocumentID = self.AIC_UUID + '_AIC_METS.xml',
                                                                       TimeZone = TimeZone)
                                if errno:
                                    logging.error('Problem to create AIC METS file, why: %s' % str(info_list))                                
                            else:
                                logging.error('Problem to get objects related to AIC_UUID: %s from DB' % (self.AIC_UUID))
                                self.ok = 0

                        if self.ok:                            
                            errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 49, 0)
                            if errno: 
                                logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                            else:
                                self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
                                logging.info(self.event_info)
                                ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'0',self.event_info,2,self.ObjectIdentifierValue)                            
                        else:
                            errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 40, 100)
                            if errno: 
                                logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                            else:
                                self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
                                logging.error(self.event_info)
                                ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                    elif self.ChecksumAlgorithm == 0: #self.ChecksumAlgorithm 0 = No checksum
                        logging.info('Skip creation of checksum: ' + self.ObjectIdentifierValue)
                        self.ObjectMessageDigest = ''
                        self.MetaObjectSize = os.stat(self.Cmets_objpath)[6] 
                        self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                        self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                        res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
                                                                                  'ObjectMessageDigest',self.ObjectMessageDigest,
                                                                                  'StatusProcess','49',
                                                                                  'StatusActivity','0',
                                                                                  'MetaObjectSize',self.MetaObjectSize,
                                                                                  'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                                  'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                                  'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                                 ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                        if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                        else: ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'0','Skip creation of checksum',2,self.ObjectIdentifierValue)
                        if errno == 0 and ExtDBupdate:
                            ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
                                                                                                     'ObjectMessageDigest',self.ObjectMessageDigest,
                                                                                                     'StatusProcess','49',
                                                                                                     'StatusActivity','0',
                                                                                                     'MetaObjectSize',self.MetaObjectSize,
                                                                                                     'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                                     'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                                    ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                            if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                            else:
                                res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                                if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                db.close_old_connections()
                self.mLock.release()
                time.sleep(int(self.Time))
        self.mDieFlag=0
Exemple #8
0
def check_storage():
    EL_root = etree.Element('needcopies')
    Policy_obj_list = ESSArchPolicy.objects.filter(PolicyStat=1).all()
    for Policy_obj in Policy_obj_list:
        sm_obj_list = [
            [
                Policy_obj.sm_1, Policy_obj.sm_type_1, Policy_obj.sm_format_1,
                Policy_obj.sm_blocksize_1, Policy_obj.sm_maxCapacity_1,
                Policy_obj.sm_minChunkSize_1, Policy_obj.sm_minContainerSize_1,
                Policy_obj.sm_target_1
            ],
            [
                Policy_obj.sm_2, Policy_obj.sm_type_2, Policy_obj.sm_format_2,
                Policy_obj.sm_blocksize_2, Policy_obj.sm_maxCapacity_2,
                Policy_obj.sm_minChunkSize_2, Policy_obj.sm_minContainerSize_2,
                Policy_obj.sm_target_2
            ],
            [
                Policy_obj.sm_3, Policy_obj.sm_type_3, Policy_obj.sm_format_3,
                Policy_obj.sm_blocksize_3, Policy_obj.sm_maxCapacity_3,
                Policy_obj.sm_minChunkSize_3, Policy_obj.sm_minContainerSize_3,
                Policy_obj.sm_target_3
            ],
            [
                Policy_obj.sm_4, Policy_obj.sm_type_4, Policy_obj.sm_format_4,
                Policy_obj.sm_blocksize_4, Policy_obj.sm_maxCapacity_4,
                Policy_obj.sm_minChunkSize_4, Policy_obj.sm_minContainerSize_4,
                Policy_obj.sm_target_4
            ],
        ]
        ip_obj_list = ArchiveObject.objects.filter(
            PolicyId=Policy_obj.PolicyID, StatusProcess=3000,
            StatusActivity=0).all()
        for ip_obj in ip_obj_list:
            storage_obj_list = ip_obj.storage_set.all()
            sm_num = 0
            for sm_obj in sm_obj_list:
                sm_num += 1
                if sm_obj[0] == 1:
                    storage_count = 0
                    storageMediumID_list = []
                    for storage_obj in storage_obj_list:
                        storageMedium_obj = storage_obj.storageMediumUUID
                        if str(sm_obj[1])[0] == '2':  #Disk
                            if storageMedium_obj.storageMedium == sm_obj[
                                    1] and storageMedium_obj.storageMediumID == 'disk':
                                storage_count += 1
                                storageMediumID_list.append(
                                    storageMedium_obj.storageMediumID)
                        elif str(sm_obj[1])[0] == '3':  #Tape
                            if storageMedium_obj.storageMedium == sm_obj[
                                    1] and storageMedium_obj.storageMediumID.startswith(
                                        sm_obj[7]):
                                storage_count += 1
                                storageMediumID_list.append(
                                    storageMedium_obj.storageMediumID)
                    if storage_count == 0:
                        EL_object = etree.SubElement(
                            EL_root,
                            'object',
                            attrib={
                                'id': ip_obj.ObjectIdentifierValue,
                                'target': sm_obj[7],
                            })
                        print 'Missing storage entry for storage method number: %s, target: %s, for object: %s' % (
                            sm_num, sm_obj[7], ip_obj.ObjectIdentifierValue)

                    elif storage_count == 1:
                        print 'Found storage entry for storage method number: %s, target: %s (%s), for object: %s' % (
                            sm_num, sm_obj[7], ','.join(
                                str(e) for e in storageMediumID_list),
                            ip_obj.ObjectIdentifierValue)
                    else:
                        print 'Warning found to many storage entry for storage method number: %s, target: %s (%s), for object: %s' % (
                            sm_num, sm_obj[7], ','.join(
                                str(e) for e in storageMediumID_list),
                            ip_obj.ObjectIdentifierValue)
    doc = etree.ElementTree(element=EL_root, file=None)
    ESSMD.writeToFile(doc, '/ESSArch/log/needcopies/needcopies.xml')
Exemple #9
0
    def ObjectValidate(self):
        self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET', ('Value', ),
                                             ('Name', 'IngestTable'))[0][0]
        if ExtDBupdate:
            self.ext_IngestTable = self.IngestTable
        else:
            self.ext_IngestTable = ''

        # Check if exist extDB and got projektid
        self.dbget, errno, why = ESSDB.DB().action(
            self.IngestTable, 'GET4',
            ('ObjectIdentifierValue', 'PolicyID', 'StatusProcess',
             'StatusActivity', 'ObjectUUID'),
            ('StatusActivity', '=', '0', 'AND', 'StatusProcess', 'BETWEEN', 9,
             'AND', 14, 'OR', 'StatusActivity', '=', '4', 'AND',
             'StatusProcess', 'BETWEEN', 10, 'AND', 11))
        if errno:
            logging.error('Failed to access Local DB, error: ' + str(why))
        for self.obj in self.dbget:
            self.ObjectIdentifierValue = self.obj[0]
            self.PolicyID = self.obj[1]
            self.StatusProcess = self.obj[2]
            self.StatusActivity = self.obj[3]
            ObjectUUID = self.obj[4]
            self.DBmode = 0
            self.ext_ObjectGuid = ObjectUUID
            self.ext_EntryDate = datetime.datetime.utcnow().replace(
                microsecond=0, tzinfo=pytz.utc)
            self.ext_EntryAgentIdentifierValue = None
            self.ext_OAISPackageType = 2
            self.ext_preservationLevelValue = 1
            self.ext_ObjectActive = 0
            self.objectstatus = 0
            self.ext_ProjectGroupCode = ''
            self.ext_ObjectPackageName = ''
            if Debug:
                logging.info('StatusProcess 9, ObjectIdentifierValue ' +
                             str(self.ObjectIdentifierValue))
            #Check....
            if self.PolicyID:
                ArchivePolicy_objs = ArchivePolicy.objects.filter(
                    PolicyStat=1, PolicyID=str(self.PolicyID))[:1]
                if ArchivePolicy_objs:
                    ArchivePolicy_obj = ArchivePolicy_objs.get()
                    if ArchivePolicy_obj.Mode in range(0, 2):
                        self.DBmode = ArchivePolicy_obj.Mode
                        logging.info(
                            'Policy found for Object: %s in ESSArch mode' %
                            self.ObjectIdentifierValue)
                        if ArchivePolicy_obj.IngestMetadata in [1, 2, 3]:
                            metsfilename = os.path.join(
                                ArchivePolicy_obj.IngestPath,
                                self.ObjectIdentifierValue +
                                '_Package_METS.xml')
                        elif ArchivePolicy_obj.IngestMetadata in [4]:
                            ObjectPath = os.path.join(
                                ArchivePolicy_obj.IngestPath,
                                self.ObjectIdentifierValue)
                            if os.path.exists(
                                    os.path.join(ObjectPath, 'sip.xml')):
                                metsfilename = os.path.join(
                                    ObjectPath, 'sip.xml')
                            elif os.path.exists(
                                    os.path.join(ObjectPath, 'mets.xml')):
                                metsfilename = os.path.join(
                                    ObjectPath, 'mets.xml')
                            #elif os.path.exists(os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
                            #    metsfilename = os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)
                            else:
                                metsfilename = ''
                            #metsfilename = '%s/sip.xml' % os.path.join(ArchivePolicy_obj.IngestPath,self.ObjectIdentifierValue)
                        else:
                            metsfilename = ''
                        res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
                            FILENAME=metsfilename)
                        if not error:
                            # cut off microsecond and timezone info ".xxxxxxx+02:00"
                            #for c in res_info[1][0]:
                            #    if c == '.' or c == '+':
                            #        break
                            #    else:
                            #        self.ext_EntryDate += c
                            self.ext_EntryDate = parse_datetime(
                                res_info[1][0]).astimezone(pytz.utc)
                            self.ext_EntryAgentIdentifierValue = res_info[2][
                                0][4]
                            try:
                                self.ext_ObjectGuid = str(
                                    uuid.UUID(self.ObjectIdentifierValue))
                            except ValueError, why:
                                logging.warning(
                                    'ObjectIdentifierValue: %s is not a valid UUID, why: %s , start to generate a new UUID'
                                    % (self.ObjectIdentifierValue, str(why)))
                                self.ext_ObjectGuid = str(uuid.uuid1())
                                logging.info(
                                    'New UUID: %s for ObjectIdentifierValue: %s'
                                    % (self.ext_ObjectGuid,
                                       str(self.ObjectIdentifierValue)))
                            self.ext_ObjectActive = 1
                            self.ext_OAISPackageType = 2
                            self.ext_preservationLevelValue = 1
                            self.objectstatus = 1
                        else:
                            self.objectstatus = 102  # Problem to get information from package METS
                    elif ArchivePolicy_obj.Mode == 2:  # AIS but POLICYID from METS, Check in AIS if object is active.
                        self.DBmode = ArchivePolicy_obj.Mode
                        self.extOBJdbget, ext_errno, ext_why = ESSMSSQL.DB(
                        ).action(self.IngestTable, 'GET3',
                                 ('ProjectGroupCode', 'ObjectPackageName',
                                  'ObjectGuid', 'ObjectActive', 'EntryDate',
                                  'EntryAgentIdentifierValue',
                                  'OAISPackageType', 'preservationLevelValue'),
                                 ('ObjectIdentifierValue',
                                  self.ObjectIdentifierValue))
                        #self.extOBJdbget = [[10,'','7283074a-00c0-11e2-a78f-002215836500',1,'2010-07-12 16:57:45','entryagent',2,1]]
                        #ext_errno = 0
                        #ext_why = 'whywhy'
                        if ext_errno:
                            logging.error('Failed to access External DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(ext_why))
                        elif self.extOBJdbget:
                            if Debug:
                                logging.info(
                                    'Found object: %s in AIS, self.extOBJdbget: %s',
                                    self.ObjectIdentifierValue,
                                    str(self.extOBJdbget))
                            self.objectstatus = 10  # Object found in external DB
                            if self.objectstatus < 100:
                                ########################################
                                # Check if object alredy have an AIP
                                if disable_ObjectPackageName:
                                    self.ext_ObjectPackageName = ''
                                else:
                                    self.ext_ObjectPackageName = self.extOBJdbget[
                                        0][1]
                                if not self.ext_ObjectPackageName:
                                    self.objectstatus = 12  # Object do not have an AIP
                                else:
                                    self.objectstatus = 112  # Object already have an AIP
                            if self.objectstatus < 100:
                                ########################################
                                # Get GUID/UUID
                                #self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2])   #When pymssql
                                self.ext_ObjectGuid = uuid.UUID(
                                    self.extOBJdbget[0][2])
                            if self.objectstatus < 100:
                                ########################################
                                # Check if object is active
                                self.ext_ObjectActive = self.extOBJdbget[0][3]
                                if self.ext_ObjectActive == 1:
                                    self.objectstatus = 13  # Object is active
                                else:
                                    self.objectstatus = 113  # Object is not active
                            if self.objectstatus < 100:
                                ########################################
                                # Check if POLICYID in local DB "METS" is equal to ProjectGroupCode in AIS
                                self.ext_ProjectGroupCode = str(
                                    self.extOBJdbget[0][0])
                                if self.ext_ProjectGroupCode == self.PolicyID:
                                    self.objectstatus = 1  # Object have an ProjectCode
                                    logging.info(
                                        'Object: %s found in AIS with correct POLICYID'
                                        % self.ObjectIdentifierValue)
                                else:
                                    self.objectstatus = 111  # Object do not have an ProjectCode
                            ########################################
                            self.ext_EntryDate = self.extOBJdbget[0][
                                4].replace(microsecond=0,
                                           tzinfo=self.tz).astimezone(pytz.utc)
                            self.ext_EntryAgentIdentifierValue = self.extOBJdbget[
                                0][5]
                            self.ext_OAISPackageType = self.extOBJdbget[0][6]
                            self.ext_preservationLevelValue = self.extOBJdbget[
                                0][7]
                            ########################################
                            # Special function only for test
                            if force_ProjectGroupCode:
                                logging.info(
                                    'Force set ProjectGroupCode for Object: %s'
                                    % self.ObjectIdentifierValue)
                                self.objectstatus = 10
                                self.ext_ProjectGroupCode = force_ProjectGroupCode
                                self.ext_ObjectPackageName = ''
                                self.ext_ObjectGuid = str(
                                    uuid.uuid1())  # updDB
                                self.ext_ObjectActive = 1  # updDB
                                self.ext_EntryDate = datetime.datetime.utcnow(
                                ).replace(microsecond=0,
                                          tzinfo=pytz.utc)  # updDB
                                self.ext_EntryAgentIdentifierValue = None  # updDB
                                self.ext_OAISPackageType = 2  # updDB
                                self.ext_preservationLevelValue = 1  # updDB
                        else:
                            self.objectstatus = 110  # Object not found in external DB
                            if Debug:
                                logging.info(
                                    'Missing object: %s in AIS, self.extOBJdbget: %s',
                                    self.ObjectIdentifierValue,
                                    str(self.extOBJdbget))
                    else:
                        self.objectstatus = 100  # Policy is not in ESSArch mode
                else:
                    self.objectstatus = 101  # Policy not found or not active
                #model.meta.Session.close()
            else:
                self.DBmode = 2  # AIS
                self.extOBJdbget, ext_errno, ext_why = ESSMSSQL.DB().action(
                    self.IngestTable, 'GET3',
                    ('ProjectGroupCode', 'ObjectPackageName', 'ObjectGuid',
                     'ObjectActive', 'EntryDate', 'EntryAgentIdentifierValue',
                     'OAISPackageType', 'preservationLevelValue'),
                    ('ObjectIdentifierValue', self.ObjectIdentifierValue))
                if ext_errno:
                    logging.error('Failed to access External DB: ' +
                                  str(self.ObjectIdentifierValue) +
                                  ' error: ' + str(ext_why))
                elif self.extOBJdbget:
                    #if not ext_errno and self.extOBJdbget:
                    if Debug:
                        logging.info(
                            'Found object: %s in AIS, self.extOBJdbget: %s',
                            self.ObjectIdentifierValue, str(self.extOBJdbget))
                    self.objectstatus = 10  # Object found in external DB
                    ########################################
                    # Check if object alredy have an AIP
                    self.ext_ProjectGroupCode = str(self.extOBJdbget[0][0])
                    if self.objectstatus < 100 and self.ext_ProjectGroupCode:
                        self.objectstatus = 11  # Object have an ProjectCode
                    else:
                        self.objectstatus = 111  # Object do not have an ProjectCode
                    ########################################
                    # Check if object alredy have an AIP
                    if disable_ObjectPackageName:
                        self.ext_ObjectPackageName = ''
                    else:
                        self.ext_ObjectPackageName = self.extOBJdbget[0][1]
                    if self.objectstatus < 100 and not self.ext_ObjectPackageName:
                        self.objectstatus = 12  # Object do not have an AIP
                    else:
                        self.objectstatus = 112  # Object already have an AIP
                    ########################################
                    # Get GUID/UUID
                    #self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2])	#When pymssql
                    self.ext_ObjectGuid = uuid.UUID(self.extOBJdbget[0][2])
                    ########################################
                    # Check if object is active
                    self.ext_ObjectActive = self.extOBJdbget[0][3]
                    if self.objectstatus < 100 and self.ext_ObjectActive == 1:
                        self.objectstatus = 13  # Object is active
                    else:
                        self.objectstatus = 113  # Object is not active

                    ########################################
                    self.ext_EntryDate = self.extOBJdbget[0][4].replace(
                        microsecond=0, tzinfo=self.tz).astimezone(pytz.utc)
                    self.ext_EntryAgentIdentifierValue = self.extOBJdbget[0][5]
                    self.ext_OAISPackageType = self.extOBJdbget[0][6]
                    self.ext_preservationLevelValue = self.extOBJdbget[0][7]
                    if Debug:
                        logging.info('ext_ProjectGroupCode is: ' +
                                     str(self.ext_ProjectGroupCode) +
                                     ' for ObjectIdentifierValue ' +
                                     str(self.ObjectIdentifierValue))
                    if force_ProjectGroupCode:
                        logging.info(
                            'Force set ProjectGroupCode for Object: %s' %
                            self.ObjectIdentifierValue)
                        self.objectstatus = 10
                        self.ext_ProjectGroupCode = force_ProjectGroupCode
                        self.ext_ObjectPackageName = ''
                        self.ext_ObjectGuid = str(uuid.uuid1())  # updDB
                        self.ext_ObjectActive = 1  # updDB
                        self.ext_EntryDate = datetime.datetime.utcnow(
                        ).replace(microsecond=0, tzinfo=pytz.utc)  # updDB
                        self.ext_EntryAgentIdentifierValue = None  # updDB
                        self.ext_OAISPackageType = 2  # updDB
                        self.ext_preservationLevelValue = 1  # updDB
                    if self.objectstatus < 100:
                        ArchivePolicy_objs = ArchivePolicy.objects.filter(
                            PolicyStat=1,
                            AISProjectID=self.ext_ProjectGroupCode)[:1]
                        if ArchivePolicy_objs:
                            self.PolicyID = ArchivePolicy_objs.get().PolicyID
                            if Debug:
                                logging.info('PolicyID: ' + str(self.PolicyID))
                            self.objectstatus = 14  # Object got PolicyID
                        else:
                            self.objectstatus = 114  # Object mising PolicyID

                else:
                    self.objectstatus = 110  # Object not found in external DB
                    if Debug:
                        logging.info(
                            'Missing object: %s in AIS, self.extOBJdbget: %s',
                            self.ObjectIdentifierValue, str(self.extOBJdbget))

            if self.objectstatus == 100:  # Policy is not in ESSArch mode
                ###################################################################
                # Policy is not in ESSArch mode(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 12, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Policy is not in ESSArch mode for Object: ' + str(
                    self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 101:  # Policy not found or not active
                ###################################################################
                # Policy not found or active(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 12, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Policy not found or active for Object: ' + str(
                    self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 102:  # Problem to get information from Package_METS
                ###################################################################
                # Problem to get information from Package_METS(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 12, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Problem to get information from Package_METS for Object: ' + str(
                    self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 110:
                ###################################################################
                #Object don't exist in extDB(10) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 10
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 10, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))

            elif self.objectstatus == 111:
                ###################################################################
                #Object don't have any projektkod in extDB(11) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 11
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 11, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))

            elif self.objectstatus == 112:
                ###################################################################
                #Object already have an AIP!!
                ###################################################################
                self.StatusProcess = 13
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 13, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(
                    self.ObjectIdentifierValue) + ' already have an AIP!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 113:
                ###################################################################
                # Object is not active!!
                ###################################################################
                self.StatusProcess = 14
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 14, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(
                    self.ObjectIdentifierValue
                ) + ' is not active in external DB!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 114:
                ###################################################################
                #Object don't have any local policy(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug:
                    logging.info(
                        'Change to StatusProcess 12, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(
                    self.ObjectIdentifierValue
                ) + ' do not have any local policy!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '1', self.event_info,
                                       self.DBmode, self.ObjectIdentifierValue)

            elif self.objectstatus == 14 or self.objectstatus == 1:
                ###################################################################
                #Object got a policy(19) and RFNext and OK(0)
                ###################################################################
                self.StatusProcess = 19
                self.StatusActivity = 0
                if Debug:
                    logging.info(
                        'Change to StatusProcess 19, ObjectIdentifierValue ' +
                        str(self.ObjectIdentifierValue))
                ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
                                       ProcVersion, '0', '', self.DBmode,
                                       self.ObjectIdentifierValue)

            logging.info(
                'objectstatus:%s,StatusProcess:%s,StatusActivity:%s,EntryDate:%s'
                % (self.objectstatus, self.StatusProcess, self.StatusActivity,
                   self.ext_EntryDate))

            if self.objectstatus:
                self.timestamp_utc = datetime.datetime.utcnow().replace(
                    microsecond=0, tzinfo=pytz.utc)
                self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                res, errno, why = ESSDB.DB().action(
                    self.IngestTable, 'UPD',
                    ('PolicyId', self.PolicyID, 'ObjectUUID',
                     self.ext_ObjectGuid, 'EntryDate',
                     self.ext_EntryDate.replace(tzinfo=None),
                     'EntryAgentIdentifierValue',
                     self.ext_EntryAgentIdentifierValue, 'StatusProcess',
                     self.StatusProcess, 'StatusActivity', self.StatusActivity,
                     'LastEventDate', self.timestamp_utc.replace(tzinfo=None),
                     'linkingAgentIdentifierValue', AgentIdentifierValue,
                     'OAISPackageType', self.ext_OAISPackageType,
                     'preservationLevelValue', self.ext_preservationLevelValue,
                     'ObjectActive', self.ext_ObjectActive, 'LocalDBdatetime',
                     self.timestamp_utc.replace(tzinfo=None)),
                    ('ObjectIdentifierValue', self.ObjectIdentifierValue))
                if errno:
                    logging.error('Failed to update Local DB: ' +
                                  str(self.ObjectIdentifierValue) +
                                  ' error: ' + str(why))
                if errno == 0 and self.ext_IngestTable:
                    ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
                        self.IngestTable, 'UPD',
                        ('PolicyId', self.PolicyID, 'StatusProcess',
                         self.StatusProcess, 'StatusActivity',
                         self.StatusActivity, 'LastEventDate',
                         self.timestamp_dst.replace(tzinfo=None),
                         'linkingAgentIdentifierValue', AgentIdentifierValue),
                        ('ObjectIdentifierValue', self.ObjectIdentifierValue))
                    if ext_errno:
                        logging.error('Failed to update External DB: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(ext_why))
                    else:
                        res, errno, why = ESSDB.DB().action(
                            self.IngestTable, 'UPD',
                            ('ExtDBdatetime',
                             self.timestamp_utc.replace(tzinfo=None)),
                            ('ObjectIdentifierValue',
                             self.ObjectIdentifierValue))
                        if errno:
                            logging.error('Failed to update Local DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(why))
Exemple #10
0
    def ThreadMain(self, ProcName):
        logging.info('Starting ' + ProcName)
        TimeZone = timezone.get_default_timezone_name()
        self.tz = pytz.timezone(TimeZone)
        METS_NAMESPACE = SchemaProfile.objects.get(
            entity='mets_namespace').value
        METS_SCHEMALOCATION = SchemaProfile.objects.get(
            entity='mets_schemalocation').value
        METS_PROFILE = SchemaProfile.objects.get(entity='mets_profile').value
        XLINK_NAMESPACE = SchemaProfile.objects.get(
            entity='xlink_namespace').value
        XSI_NAMESPACE = SchemaProfile.objects.get(entity='xsi_namespace').value
        while 1:
            if self.mDieFlag == 1: break  # Request for death
            self.mLock.acquire()
            self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
                                                    ('Time', 'Run'),
                                                    ('Name', ProcName))[0]
            if self.Run == '0':
                logging.info('Stopping ' + ProcName)
                ESSDB.DB().action('ESSProc', 'UPD',
                                  ('Status', '0', 'Run', '0', 'PID', '0'),
                                  ('Name', ProcName))
                self.RunFlag = 0
                self.mLock.release()
                if Debug: logging.info('RunFlag: 0')
                time.sleep(2)
                continue
            # Process Item
            lock = thread.allocate_lock()
            Cmets_obj = Parameter.objects.get(
                entity='content_descriptionfile').value
            self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
                                                 ('Value', ),
                                                 ('Name', 'IngestTable'))[0][0]
            if ExtDBupdate:
                self.ext_IngestTable = self.IngestTable
            else:
                self.ext_IngestTable = ''
            self.dbget, errno, why = ESSDB.DB().action(
                self.IngestTable, 'GET4',
                ('ObjectIdentifierValue', 'ObjectUUID', 'PolicyId',
                 'ObjectSize'), ('StatusProcess', 'BETWEEN', 39, 'AND', 40,
                                 'AND', 'StatusActivity', '=', '0'))
            if errno:
                logging.error('Failed to access Local DB, error: ' + str(why))
            for self.obj in self.dbget:
                self.ok = 1
                self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
                                                ('Run', 'Pause'),
                                                ('Name', ProcName))[0]
                if self.ProcDB[0] == '0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc', 'UPD',
                                      ('Status', '0', 'Run', '0', 'PID', '0'),
                                      ('Name', ProcName))
                    thread.interrupt_main()
                    time.sleep(5)
                    break
                elif self.ProcDB[1] == 1:
                    while 1:
                        time.sleep(60)
                        self.ProcDB = ESSDB.DB().action(
                            'ESSProc', 'GET', ('Run', 'Pause'),
                            ('Name', ProcName))[0]
                        if self.ProcDB[1] == 1:
                            logging.info('Process is in pause state')
                        else:
                            break
                self.ObjectIdentifierValue = self.obj[0]
                self.ObjectUUID = self.obj[1]
                self.PolicyId = self.obj[2]
                self.ObjectSize = self.obj[3]
                ArchivePolicy_obj = ArchivePolicy.objects.get(
                    PolicyStat=1, PolicyID=self.PolicyId)
                if self.ok:
                    ###########################################################
                    # set variables
                    self.AIPpath = ArchivePolicy_obj.AIPpath
                    self.metatype = ArchivePolicy_obj.IngestMetadata
                    self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
                    self.CA = dict(ChecksumAlgorithm_CHOICES)[
                        self.ChecksumAlgorithm]
                    self.SIPpath = ArchivePolicy_obj.IngestPath
                    self.p_obj = self.ObjectIdentifierValue + '.tar'
                    self.ObjectPath = os.path.join(self.AIPpath, self.p_obj)
                    self.SIProotpath = os.path.join(self.SIPpath,
                                                    self.ObjectIdentifierValue)
                    if self.metatype in [4]:
                        #self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                        #self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
                        #self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
                        self.Cmets_obj = Cmets_obj.replace(
                            '{objid}', self.ObjectIdentifierValue)
                        self.Cmets_objpath = os.path.join(
                            self.SIProotpath, self.Cmets_obj)
                    elif self.metatype in [1, 2, 3]:
                        self.Cmets_obj = '%s_Content_METS.xml' % (
                            self.ObjectIdentifierValue)
                        self.Cmets_objpath = os.path.join(
                            self.AIPpath, self.Cmets_obj)
                    self.Pmets_obj = '%s_Package_METS.xml' % (
                        self.ObjectIdentifierValue)
                    self.Pmets_objpath = os.path.join(self.AIPpath,
                                                      self.Pmets_obj)
                    self.AIC_UUID = None
                    self.AIC_UUID_rel_ObjectUUIDs = []
                if self.ok:
                    METS_agent_list = []
                    METS_altRecordID_list = []
                    if self.metatype == 1:
                        ############################################
                        # Object have metatype 1 (METS)
                        self.METS_LABEL = 'ESSArch AIP'
                        # Get SIP Content METS information
                        self.METSfilepath = os.path.join(
                            self.SIPpath,
                            self.ObjectIdentifierValue + '/metadata/SIP/' +
                            self.ObjectIdentifierValue + '_Content_METS.xml')
                        res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
                            FILENAME=self.METSfilepath)
                        for agent in res_info[2]:
                            if not (agent[0] == 'CREATOR'
                                    and agent[3] == 'SOFTWARE'):
                                METS_agent_list.append(agent)
                        METS_agent_list.append([
                            'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
                            []
                        ])
                        METS_agent_list.append([
                            'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
                            ['VERSION=%s' % ProcVersion]
                        ])
                    elif self.metatype == 2:
                        ############################################
                        # Object have metatype 2 (RES)
                        self.METS_LABEL = 'Imaging AIP RA'
                        METS_agent_list.append([
                            'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
                        ])
                        METS_agent_list.append(
                            ['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
                        METS_agent_list.append([
                            'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
                            []
                        ])
                        METS_agent_list.append([
                            'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
                            ['VERSION=%s' % ProcVersion]
                        ])
                    elif self.metatype == 3:
                        ############################################
                        # Object have metatype 3 (ADDML)
                        self.METS_LABEL = 'Born Digital AIP RA'
                        METS_agent_list.append([
                            'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
                        ])
                        METS_agent_list.append(
                            ['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
                        METS_agent_list.append([
                            'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
                            []
                        ])
                        METS_agent_list.append([
                            'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
                            ['VERSION=%s' % ProcVersion]
                        ])
                    elif self.metatype in [4]:
                        ############################################
                        # Object have metatype 4 (eARD METS)
                        res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
                            FILENAME=self.Cmets_objpath)
                        for agent in res_info[2]:
                            #if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
                            METS_agent_list.append(agent)
                        self.METS_LABEL = res_info[0][0]
                        METS_agent_list.append([
                            'CREATOR', None, 'INDIVIDUAL', None,
                            AgentIdentifierValue, []
                        ])
                        METS_agent_list.append([
                            'CREATOR', None, 'OTHER', 'SOFTWARE', 'ESSArch',
                            ['VERSION=%s' % ProcVersion]
                        ])
                        for altRecordID in res_info[3]:
                            METS_altRecordID_list.append(altRecordID)
                logging.debug('self.obj: ' + str(self.obj))
                if self.ChecksumAlgorithm > 0:  #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
                    self.startCalTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    errno, why = ESSPGM.DB().SetAIPstatus(
                        self.IngestTable, self.ext_IngestTable,
                        AgentIdentifierValue, self.ObjectUUID, 40, 5)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    logging.info('Start create Package METS for: ' +
                                 self.ObjectIdentifierValue)
                    if self.ok:
                        ###########################################################
                        # Create PMETS for AIP package
                        self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(
                            self.Cmets_objpath, self.CA)
                        if errno:
                            self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(
                                self.Cmets_objpath)
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1030', '',
                                                   'ESSArch AIPCreator',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        self.M_statinfo = os.stat(self.Cmets_objpath)
                        self.M_SIZE = self.M_statinfo.st_size
                        self.M_utc_mtime = datetime.datetime.utcfromtimestamp(
                            self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                        self.M_lociso_mtime = self.M_utc_mtime.astimezone(
                            self.tz).isoformat()
                        self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(
                            self.ObjectPath, self.CA)
                        if errno:
                            self.event_info = 'Problem to get checksum for AIP package: ' + str(
                                self.ObjectPath)
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1040', '',
                                                   'ESSArch AIPChecksum',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                            self.ok = 0
                        self.P_statinfo = os.stat(self.ObjectPath)
                        self.P_SIZE = self.P_statinfo.st_size
                        self.P_utc_mtime = datetime.datetime.utcfromtimestamp(
                            self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                        self.P_lociso_mtime = self.P_utc_mtime.astimezone(
                            self.tz).isoformat()

                        if self.metatype in [1, 2, 3]:
                            self.PMETSdoc = ESSMD.createPMets(
                                ID=self.ObjectIdentifierValue,
                                LABEL=self.METS_LABEL,
                                AGENT=METS_agent_list,
                                P_SIZE=self.P_SIZE,
                                P_CREATED=self.P_lociso_mtime,
                                P_CHECKSUM=self.P_CHECKSUM,
                                P_CHECKSUMTYPE=self.CA,
                                M_SIZE=self.M_SIZE,
                                M_CREATED=self.M_lociso_mtime,
                                M_CHECKSUM=self.M_CHECKSUM,
                                M_CHECKSUMTYPE=self.CA,
                            )
                            errno, why = ESSMD.writeToFile(
                                self.PMETSdoc, self.Pmets_objpath)
                            if errno:
                                self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(
                                    self.Pmets_objpath)
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1040', '', 'ESSArch AIPChecksum',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                time.sleep(2)
                                self.ok = 0
                        elif self.metatype in [4]:
                            ms_files = []
                            ms_files.append([
                                'amdSec', None, 'techMD', 'techMD001', None,
                                None,
                                'ID%s' % str(uuid.uuid1()), 'URL',
                                'file:%s/%s' %
                                (self.ObjectIdentifierValue, self.Cmets_obj),
                                'simple', self.M_CHECKSUM, self.CA,
                                self.M_SIZE, 'text/xml', self.M_lociso_mtime,
                                'OTHER', 'METS', None
                            ])

                            ms_files.append([
                                'fileSec', None, None, None, None, None,
                                'ID%s' % str(uuid.uuid1()), 'URL',
                                'file:%s' % self.p_obj, 'simple',
                                self.P_CHECKSUM, self.CA, self.P_SIZE,
                                'application/x-tar', self.P_lociso_mtime,
                                'tar', 'techMD001', None
                            ])
                            # define namespaces
                            self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
                            self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
                            self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
                            self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
                                METS_NAMESPACE, METS_SCHEMALOCATION)

                            errno, info_list = ESSMD.Create_IP_mets(
                                ObjectIdentifierValue=self.
                                ObjectIdentifierValue,
                                METS_ObjectPath=self.Pmets_objpath,
                                agent_list=METS_agent_list,
                                altRecordID_list=METS_altRecordID_list,
                                file_list=ms_files,
                                namespacedef=self.namespacedef,
                                METS_LABEL=self.METS_LABEL,
                                METS_PROFILE=METS_PROFILE,
                                METS_TYPE='AIP',
                                METS_DocumentID=self.Pmets_obj,
                                TimeZone=TimeZone)
                            if errno:
                                logging.error(
                                    'Problem to create Package METS file, why: %s'
                                    % str(info_list))

                    self.ObjectMessageDigest = self.P_CHECKSUM
                    self.stopCalTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    self.CalTime = self.stopCalTime - self.startCalTime
                    self.ObjectSizeMB = self.ObjectSize / 1048576
                    if self.CalTime.seconds < 1:
                        self.CalTime = datetime.timedelta(
                            seconds=1
                        )  #Fix min time to 1 second if it is zero.
                    self.CalMBperSEC = int(self.ObjectSizeMB) / int(
                        self.CalTime.seconds)
                    logging.info('Finished calculate checksum: ' +
                                 self.ObjectIdentifierValue + ' , ' +
                                 str(self.CalMBperSEC) + ' MB/Sec and Time: ' +
                                 str(self.CalTime))

                    if self.ok:
                        self.timestamp_utc = datetime.datetime.utcnow(
                        ).replace(microsecond=0, tzinfo=pytz.utc)
                        self.timestamp_dst = self.timestamp_utc.astimezone(
                            self.tz)
                        res, errno, why = ESSDB.DB().action(
                            self.IngestTable, 'UPD',
                            ('ObjectMessageDigestAlgorithm',
                             self.ChecksumAlgorithm, 'ObjectMessageDigest',
                             self.ObjectMessageDigest, 'MetaObjectSize',
                             self.M_SIZE, 'LastEventDate',
                             self.timestamp_utc.replace(tzinfo=None),
                             'linkingAgentIdentifierValue',
                             AgentIdentifierValue, 'LocalDBdatetime',
                             self.timestamp_utc.replace(tzinfo=None)),
                            ('ObjectIdentifierValue',
                             self.ObjectIdentifierValue))
                        if errno:
                            logging.error('Failed to update Local DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(why))
                        if errno == 0 and ExtDBupdate:
                            ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
                                self.IngestTable, 'UPD',
                                ('ObjectMessageDigestAlgorithm',
                                 self.ChecksumAlgorithm, 'ObjectMessageDigest',
                                 self.ObjectMessageDigest, 'MetaObjectSize',
                                 self.M_SIZE, 'LastEventDate',
                                 self.timestamp_dst.replace(tzinfo=None),
                                 'linkingAgentIdentifierValue',
                                 AgentIdentifierValue),
                                ('ObjectIdentifierValue',
                                 self.ObjectIdentifierValue))
                            if ext_errno:
                                logging.error(
                                    'Failed to update External DB: ' +
                                    str(self.ObjectIdentifierValue) +
                                    ' error: ' + str(ext_why))
                            else:
                                res, errno, why = ESSDB.DB().action(
                                    self.IngestTable, 'UPD',
                                    ('ExtDBdatetime',
                                     self.timestamp_utc.replace(tzinfo=None)),
                                    ('ObjectIdentifierValue',
                                     self.ObjectIdentifierValue))
                                if errno:
                                    logging.error(
                                        'Failed to update Local DB: ' +
                                        str(self.ObjectIdentifierValue) +
                                        ' error: ' + str(why))

                    if self.ok and self.metatype == 4:
                        ####################################################
                        # Create AIC METS File:
                        aic_obj = ArchiveObject.objects.filter(
                            relaic_set__UUID=self.ObjectUUID)[:1]
                        if aic_obj:
                            self.AIC_UUID = aic_obj.get().ObjectUUID
                            logging.info(
                                'Succeeded to get AIC_UUID: %s from DB' %
                                self.AIC_UUID)
                        else:
                            logging.warning(
                                'AIC not found for IP object: %s, skip to create AIC METS file'
                                % self.ObjectUUID)
                    if self.ok and self.AIC_UUID:
                        ip_obj_list = ArchiveObject.objects.filter(
                            Q(StatusProcess=3000)
                            | Q(ObjectUUID=self.ObjectUUID),
                            reluuid_set__AIC_UUID=self.AIC_UUID).order_by(
                                'Generation')
                        if ip_obj_list:
                            logging.info('Start create AIC METS: ' +
                                         self.AIC_UUID)
                            self.AICmets_objpath = os.path.join(
                                self.AIPpath, self.AIC_UUID + '_AIC_METS.xml')
                            ms_files = []
                            for ip_obj in ip_obj_list:
                                logging.info(
                                    'Add IP: %s to AIC METS: %s' %
                                    (ip_obj.ObjectUUID, self.AIC_UUID))
                                ms_files.append([
                                    'fileSec', None, None, None, None, None,
                                    'ID%s' % str(uuid.uuid1()), 'URL',
                                    'file:%s' % ip_obj.ObjectUUID, 'simple',
                                    ip_obj.ObjectMessageDigest,
                                    dict(ChecksumAlgorithm_CHOICES)[
                                        ip_obj.ObjectMessageDigestAlgorithm],
                                    ip_obj.ObjectSize, 'application/x-tar',
                                    ip_obj.CreateDate, 'IP Package', None, None
                                ])

                            # define namespaces
                            self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
                            self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
                            self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
                            self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
                                METS_NAMESPACE, METS_SCHEMALOCATION)

                            errno, info_list = ESSMD.Create_IP_mets(
                                ObjectIdentifierValue=self.AIC_UUID,
                                METS_ObjectPath=self.AICmets_objpath,
                                agent_list=[],
                                altRecordID_list=[],
                                file_list=ms_files,
                                namespacedef=self.namespacedef,
                                METS_LABEL='AIC relation to IP',
                                METS_PROFILE=METS_PROFILE,
                                METS_TYPE='AIC',
                                METS_DocumentID=self.AIC_UUID +
                                '_AIC_METS.xml',
                                TimeZone=TimeZone)
                            if errno:
                                logging.error(
                                    'Problem to create AIC METS file, why: %s'
                                    % str(info_list))
                        else:
                            logging.error(
                                'Problem to get objects related to AIC_UUID: %s from DB'
                                % (self.AIC_UUID))
                            self.ok = 0

                    if self.ok:
                        errno, why = ESSPGM.DB().SetAIPstatus(
                            self.IngestTable, self.ext_IngestTable,
                            AgentIdentifierValue, self.ObjectUUID, 49, 0)
                        if errno:
                            logging.error(
                                'Failed to update DB status for AIP: ' +
                                str(self.ObjectIdentifierValue) + ' error: ' +
                                str(why))
                        else:
                            self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
                            logging.info(self.event_info)
                            ESSPGM.Events().create('1040', '',
                                                   'ESSArch AIPChecksum',
                                                   ProcVersion, '0',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                    else:
                        errno, why = ESSPGM.DB().SetAIPstatus(
                            self.IngestTable, self.ext_IngestTable,
                            AgentIdentifierValue, self.ObjectUUID, 40, 100)
                        if errno:
                            logging.error(
                                'Failed to update DB status for AIP: ' +
                                str(self.ObjectIdentifierValue) + ' error: ' +
                                str(why))
                        else:
                            self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
                            logging.error(self.event_info)
                            ESSPGM.Events().create('1040', '',
                                                   'ESSArch AIPChecksum',
                                                   ProcVersion, '1',
                                                   self.event_info, 2,
                                                   self.ObjectIdentifierValue)
                elif self.ChecksumAlgorithm == 0:  #self.ChecksumAlgorithm 0 = No checksum
                    logging.info('Skip creation of checksum: ' +
                                 self.ObjectIdentifierValue)
                    self.ObjectMessageDigest = ''
                    self.MetaObjectSize = os.stat(self.Cmets_objpath)[6]
                    self.timestamp_utc = datetime.datetime.utcnow().replace(
                        microsecond=0, tzinfo=pytz.utc)
                    self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                    res, errno, why = ESSDB.DB().action(
                        self.IngestTable, 'UPD',
                        ('ObjectMessageDigestAlgorithm',
                         self.ChecksumAlgorithm, 'ObjectMessageDigest',
                         self.ObjectMessageDigest, 'StatusProcess', '49',
                         'StatusActivity', '0', 'MetaObjectSize',
                         self.MetaObjectSize, 'LastEventDate',
                         self.timestamp_utc.replace(tzinfo=None),
                         'linkingAgentIdentifierValue', AgentIdentifierValue,
                         'LocalDBdatetime',
                         self.timestamp_utc.replace(tzinfo=None)),
                        ('ObjectIdentifierValue', self.ObjectIdentifierValue))
                    if errno:
                        logging.error('Failed to update Local DB: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    else:
                        ESSPGM.Events().create('1040', '',
                                               'ESSArch AIPChecksum',
                                               ProcVersion, '0',
                                               'Skip creation of checksum', 2,
                                               self.ObjectIdentifierValue)
                    if errno == 0 and ExtDBupdate:
                        ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
                            self.IngestTable, 'UPD',
                            ('ObjectMessageDigestAlgorithm',
                             self.ChecksumAlgorithm, 'ObjectMessageDigest',
                             self.ObjectMessageDigest, 'StatusProcess', '49',
                             'StatusActivity', '0', 'MetaObjectSize',
                             self.MetaObjectSize, 'LastEventDate',
                             self.timestamp_dst.replace(tzinfo=None),
                             'linkingAgentIdentifierValue',
                             AgentIdentifierValue),
                            ('ObjectIdentifierValue',
                             self.ObjectIdentifierValue))
                        if ext_errno:
                            logging.error('Failed to update External DB: ' +
                                          str(self.ObjectIdentifierValue) +
                                          ' error: ' + str(ext_why))
                        else:
                            res, errno, why = ESSDB.DB().action(
                                self.IngestTable, 'UPD',
                                ('ExtDBdatetime',
                                 self.timestamp_utc.replace(tzinfo=None)),
                                ('ObjectIdentifierValue',
                                 self.ObjectIdentifierValue))
                            if errno:
                                logging.error('Failed to update Local DB: ' +
                                              str(self.ObjectIdentifierValue) +
                                              ' error: ' + str(why))
            db.close_old_connections()
            self.mLock.release()
            time.sleep(int(self.Time))
        self.mDieFlag = 0
    def ThreadMain(self, ProcName):
        logging.info('Starting ' + ProcName)
        while 1:
            if self.mDieFlag == 1: break  # Request for death
            self.mLock.acquire()
            self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
                                                    ('Time', 'Run'),
                                                    ('Name', ProcName))[0]
            if self.Run == '0':
                logging.info('Stopping ' + ProcName)
                ESSDB.DB().action('ESSProc', 'UPD',
                                  ('Status', '0', 'Run', '0', 'PID', '0'),
                                  ('Name', ProcName))
                self.RunFlag = 0
                self.mLock.release()
                #if Debug: print 'RunFlag: 0'
                time.sleep(2)
                continue
            # Process Item
            lock = thread.allocate_lock()
            self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
                                                 ('Value', ),
                                                 ('Name', 'IngestTable'))[0][0]
            if ExtDBupdate:
                self.ext_IngestTable = self.IngestTable
            else:
                self.ext_IngestTable = ''
            #if Debug: logging.info('Start to list worklist (self.dbget)')

            self.dbget, errno, why = ESSDB.DB().action(
                self.IngestTable, 'GET4',
                ('ObjectIdentifierValue', 'ObjectUUID', 'PolicyId',
                 'INFORMATIONCLASS'),
                ('StatusActivity', '=', '0', 'AND', 'StatusProcess', 'BETWEEN',
                 24, 'AND', 26))
            if errno:
                logging.error('Failed to access Local DB, error: ' + str(why))
            for self.obj in self.dbget:
                self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
                                                ('Run', 'Pause'),
                                                ('Name', ProcName))[0]
                if self.ProcDB[0] == '0':
                    logging.info('Stopping ' + ProcName)
                    ESSDB.DB().action('ESSProc', 'UPD',
                                      ('Status', '0', 'Run', '0', 'PID', '0'),
                                      ('Name', ProcName))
                    thread.interrupt_main()
                    time.sleep(5)
                    break
                elif self.ProcDB[1] == 1:
                    while 1:
                        time.sleep(60)
                        self.ProcDB = ESSDB.DB().action(
                            'ESSProc', 'GET', ('Run', 'Pause'),
                            ('Name', ProcName))[0]
                        if self.ProcDB[1] == 1:
                            logging.info('Process is in pause state')
                        else:
                            break
                self.ok = 1
                ###########################################################
                # get policy info
                self.ObjectIdentifierValue = ESSPGM.Check().str2unicode(
                    self.obj[0])
                self.ObjectUUID = self.obj[1]
                self.PolicyId = self.obj[2]
                self.INFORMATIONCLASS = self.obj[3]
                logging.info('Start to validate format for SIP: %s',
                             self.ObjectIdentifierValue)
                self.ChecksumAlgorithm_CHOICES_dict = dict(
                    ChecksumAlgorithm_CHOICES)
                self.ChecksumAlgorithm_CHOICES_invdict = ESSPGM.Check(
                ).invert_dict(self.ChecksumAlgorithm_CHOICES_dict)
                ArchivePolicy_obj = ArchivePolicy.objects.get(
                    PolicyStat=1, PolicyID=self.PolicyId)
                if self.ok:
                    ###########################################################
                    # set variables
                    self.AIPpath = ESSPGM.Check().str2unicode(
                        ArchivePolicy_obj.AIPpath)
                    self.metatype = ArchivePolicy_obj.IngestMetadata
                    self.Policy_INFORMATIONCLASS = ArchivePolicy_obj.INFORMATIONCLASS
                    self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
                    self.ChecksumAlgorithm_name = self.ChecksumAlgorithm_CHOICES_dict[
                        self.ChecksumAlgorithm]
                    self.SIPpath = ESSPGM.Check().str2unicode(
                        ArchivePolicy_obj.IngestPath)
                    self.DBmode = ArchivePolicy_obj.Mode
                    logging.debug('self.obj: %s', str(self.obj))
                    logging.debug('self.ObjectIdentifierValue: %s',
                                  self.ObjectIdentifierValue)
                    logging.debug('Len self.ObjectIdentifierValue: %s',
                                  len(self.ObjectIdentifierValue))
                    logging.debug('self.SIPpath: %s', self.SIPpath)
                    logging.debug('self.AIPpath: %s', self.AIPpath)
                if self.metatype == 2:
                    ############################################
                    # Create PREMISfile from TIFFEdit.RES if metatype is 2
                    logging.info(
                        'Start to convert RESfile to PREMISfile for object: ' +
                        self.ObjectIdentifierValue)
                    self.xml_PREMIS, self.errno, self.why = ESSMD.RES2PREMIS(
                        os.path.join(self.SIPpath, self.ObjectIdentifierValue),
                        AgentIdentifierValue[8:])
                    if self.errno == 10:
                        self.event_info = 'Failed to parse RESfile, error.num: %s error.det: %s' % (
                            str(self.errno), str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                    elif self.errno == 20:
                        self.event_info = 'I/O error to access RESfile, error.num: %s error.det: %s' % (
                            str(self.errno), str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                    elif self.errno == 30:
                        self.event_info = 'Validation errors for PREMIS file, error.num: %s error.det: %s' % (
                            str(self.errno), str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                    elif self.errno == 40:
                        self.event_info = 'Problem to write PREMIS file, error.num: %s error.det: %s' % (
                            str(self.errno), str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                    if self.errno > 1:
                        self.event_info = 'Problem to convert RES to PREMIS for SIP package: %s, error.num: %s  error.desc: %s' % (
                            self.ObjectIdentifierValue, str(
                                self.errno), str(self.why))
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                        self.ok = 0
                    elif self.errno == 1:
                        self.event_info = 'Warning in convert RES to PREMIS for SIP package: %s, error.num: %s  warning.desc: %s' % (
                            self.ObjectIdentifierValue, str(
                                self.errno), str(self.why))
                        logging.warning(self.event_info)
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '0',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
                    else:
                        ESSPGM.Events().create('1022', 'RES2PREMIS',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '0', '',
                                               self.DBmode,
                                               self.ObjectIdentifierValue)
                    if self.ok:
                        ############################################
                        # Clean RES SIP from "junk" files
                        self.errno, self.why = ESSPGM.Check().CleanRES_SIP(
                            os.path.join(self.SIPpath,
                                         self.ObjectIdentifierValue))
                        if self.errno:
                            self.event_info = 'Problem to clean RES SIP from "junk files" for SIP package: %s, error.num: %s  error.desc: %s' % (
                                self.ObjectIdentifierValue, str(
                                    self.errno), str(self.why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1022', 'CleanRES_SIP',
                                'ESSArch SIPValidateFormat', ProcVersion, '1',
                                self.event_info, self.DBmode,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                        else:
                            ESSPGM.Events().create(
                                '1022', 'CleanRES_SIP',
                                'ESSArch SIPValidateFormat', ProcVersion, '0',
                                '', self.DBmode, self.ObjectIdentifierValue)
                elif self.metatype == 1:
                    ###########################################################
                    # Create PREMISfile from Content_METS if metatype is 1
                    res, errno, why = ESSMD.METS2PREMIS(
                        self.SIPpath, self.ObjectIdentifierValue)
                    if not errno:
                        logging.info(
                            'Succeeded to convert Content_METS to PREMISfile for information package: %s',
                            self.ObjectIdentifierValue)
                    else:
                        self.event_info = 'Problem to convert Content_METS to PREMISfile for information package: %s, errno: %s, detail: %s' % (
                            self.ObjectIdentifierValue, str(errno), str(why))
                        logging.error(self.event_info)
                        #ESSPGM.Events().create('1025','','ESSArch SIPValidateFormat',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)
                        self.ok = 0
                elif self.metatype in [4]:
                    self.SIPpath = os.path.join(self.SIPpath,
                                                self.ObjectIdentifierValue)
                if self.ok:
                    if self.metatype in [1, 2, 3]:
                        ###########################################################
                        # get object_list from PREMIS file
                        self.Premis_filepath = u'%s/%s/%s_PREMIS.xml' % (
                            self.SIPpath, self.ObjectIdentifierValue,
                            self.ObjectIdentifierValue)
                        self.object_list, errno, why = ESSMD.getPremisObjects(
                            FILENAME=self.Premis_filepath)
                        if errno == 0:
                            logging.info(
                                'Succeeded to get object_list from premis for information package: %s',
                                self.ObjectIdentifierValue)
                        else:
                            self.event_info = 'Problem to get object_list from premis for information package: %s, errno: %s, detail: %s' % (
                                self.ObjectIdentifierValue, str(errno),
                                str(why))
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1025', '', 'ESSArch SIPValidateFormat',
                                ProcVersion, '1', self.event_info, self.DBmode,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                    elif self.metatype in [4]:
                        ###########################################################
                        # get object_list from METS
                        if os.path.exists(os.path.join(self.SIPpath,
                                                       'sip.xml')):
                            mets_file = 'sip.xml'
                            self.SIPmets_objpath = os.path.join(
                                self.SIPpath, mets_file)
                        elif os.path.exists(
                                os.path.join(self.SIPpath, 'mets.xml')):
                            mets_file = 'mets.xml'
                            self.SIPmets_objpath = os.path.join(
                                self.SIPpath, mets_file)
                        #elif os.path.exists(os.path.join(self.SIPpath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
                        #    mets_file = '%s_Content_METS.xml' % self.ObjectIdentifierValue
                        #    self.SIPmets_objpath = os.path.join(self.SIPpath,mets_file)
                        else:
                            self.SIPmets_objpath = ''
                            self.event_info = 'Problem to find METS file for information package: %s in path: %s' % (
                                self.ObjectIdentifierValue, self.SIPpath)
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1025', '', 'ESSArch SIPValidateFormat',
                                ProcVersion, '1', self.event_info, 2,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                        if self.SIPmets_objpath:
                            self.object_list, errno, why = ESSMD.getAIPObjects(
                                FILENAME=self.SIPmets_objpath)
                            if errno == 0:
                                logging.info(
                                    'Succeeded to get object_list from METS for information package: %s',
                                    self.ObjectIdentifierValue)
                                self.F_Checksum, errno, why = ESSPGM.Check(
                                ).checksum(self.SIPmets_objpath,
                                           self.ChecksumAlgorithm)  # Checksum
                                self.F_SIZE = os.stat(self.SIPmets_objpath)[6]
                                self.object_list.append([
                                    mets_file, self.ChecksumAlgorithm_name,
                                    self.F_Checksum, self.F_SIZE, ''
                                ])
                            else:
                                self.event_info = 'Problem to get object_list from METS for information package: %s, errno: %s, detail: %s' % (
                                    self.ObjectIdentifierValue, str(errno),
                                    str(why))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1025', '', 'ESSArch SIPValidateFormat',
                                    ProcVersion, '1', self.event_info, 2,
                                    self.ObjectIdentifierValue)
                                self.ok = 0
                if self.ok:
                    ###########################################################
                    # update ObjectIdentifierValue to StatusProcess: 25 and StatusActivity: 5
                    errno, why = ESSPGM.DB().SetAIPstatus(
                        self.IngestTable, self.ext_IngestTable,
                        AgentIdentifierValue, self.ObjectUUID, 25, 5)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    logging.info('Format validate object: ' +
                                 self.ObjectIdentifierValue)
                if self.ok:
                    ###########################################################
                    # Start to format validate SIP
                    self.startTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    self.ObjectNumItems = 0
                    self.ObjectSize = 0
                    if self.metatype == 1:
                        ############################################
                        # Object have metatype 1 (METS)
                        self.tmp_object_id = (
                            u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,
                                                    self.ObjectIdentifierValue)
                        self.object_list.append([
                            self.tmp_object_id, '', '', '', '',
                            'ARCHMETAxmlWrap', 'PREMIS'
                        ])
                    elif self.metatype == 2:
                        ############################################
                        # Object have metatype 2 (RES)
                        #self.tmp_object_id = ('%s/TIFFEdit.RES') % self.ObjectIdentifierValue
                        #self.object_list.append([self.tmp_object_id,'', '', '', '', 'ARCHMETA', ''])
                        self.tmp_object_id = (
                            u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,
                                                    self.ObjectIdentifierValue)
                        self.object_list.append([
                            self.tmp_object_id, '', '', '', '',
                            'ARCHMETAxmlWrap', 'PREMIS'
                        ])
                    elif self.metatype == 3:
                        ############################################
                        # Object have metatype 3 (ADDML)
                        self.tmp_object_id = (
                            u'%s/%s_ADDML.xml') % (self.ObjectIdentifierValue,
                                                   self.ObjectIdentifierValue)
                        self.object_list.append([
                            self.tmp_object_id, '', '', '', '',
                            'ARCHMETAxmlWrap', 'ADDML'
                        ])
                        self.tmp_object_id = (
                            u'%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,
                                                    self.ObjectIdentifierValue)
                        self.object_list.append([
                            self.tmp_object_id, '', '', '', '',
                            'ARCHMETAxmlWrap', 'PREMIS'
                        ])
                    for self.object in self.object_list:
                        logging.debug('variable self.SIPpath: %s, type: %s' %
                                      (self.SIPpath, type(self.SIPpath)))
                        logging.debug('variable self.object[0]: %s, type: %s' %
                                      (self.object[0], type(self.object[0])))
                        self.filepath = os.path.join(self.SIPpath,
                                                     self.object[0])
                        logging.debug('variable self.filepath: %s, type: %s' %
                                      (self.filepath, type(self.filepath)))
                        #self.filepath = ESSPGM.Check().Unicode2isoStr(self.filepath.encode('utf-8'))
                        #self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
                        #logging.debug('variable self.filepath_iso: %s, type: %s' % (self.filepath_iso,type(self.filepath_iso)))
                        if self.metatype in [1, 2, 3
                                             ] and self.ObjectNumItems == 0:
                            if self.object[0] == self.ObjectIdentifierValue:
                                logging.info(
                                    'First premis object match information package: %s',
                                    self.ObjectIdentifierValue)
                            else:
                                self.event_info = 'First premis object do not match information package: %s, premis_object: %s' % (
                                    self.ObjectIdentifierValue, self.object[0])
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1025', '', 'ESSArch SIPValidateFormat',
                                    ProcVersion, '1', self.event_info,
                                    self.DBmode, self.ObjectIdentifierValue)
                                self.ok = 0
                                break
                            if self.ok and os.access(self.filepath, os.X_OK):
                                pass
                            else:
                                self.event_info = 'Object path: %s do not exist or is not executable!' % self.filepath
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1025', '', 'ESSArch SIPValidateFormat',
                                    ProcVersion, '1', self.event_info,
                                    self.DBmode, self.ObjectIdentifierValue)
                                self.ok = 0
                                break
                        if self.ok and os.access(self.filepath, os.R_OK):
                            pass
                        else:
                            self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1025', '', 'ESSArch SIPValidateFormat',
                                ProcVersion, '1', self.event_info, self.DBmode,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                            break
                        if self.ok and os.access(self.filepath, os.W_OK):
                            pass
                        else:
                            self.event_info = 'Missing permission, Object path: %s is not writeable!' % self.filepath
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1025', '', 'ESSArch SIPValidateFormat',
                                ProcVersion, '1', self.event_info, self.DBmode,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                            break
                        if self.metatype in [1, 2, 3]:
                            if self.ok and not (self.ObjectNumItems == 0
                                                or self.object[5] == 'ARCHMETA'
                                                or self.object[5]
                                                == 'ARCHMETAxmlWrap'):
                                if int(os.stat(self.filepath)[6]) == int(
                                        self.object[4]):
                                    self.ObjectSize += int(self.object[4])
                                else:
                                    self.event_info = 'Filesize for object path: %s is %s and premis object size is %s. The sizes must match!' % (
                                        self.filepath,
                                        str(os.stat(self.filepath)[6]),
                                        str(self.object[4]))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1025', '',
                                        'ESSArch SIPValidateFormat',
                                        ProcVersion, '1', self.event_info,
                                        self.DBmode,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                                if self.ok:
                                    self.F_Checksum, errno, why = ESSPGM.Check(
                                    ).checksum(self.filepath,
                                               self.object[1])  # Checksum
                                    if errno:
                                        self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
                                            self.filepath, str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create(
                                            '1025', '',
                                            'ESSArch SIPValidateFormat',
                                            ProcVersion, '1', self.event_info,
                                            self.DBmode,
                                            self.ObjectIdentifierValue)
                                        self.ok = 0
                                if self.ok:
                                    if self.F_Checksum == self.object[2]:
                                        pass
                                    else:
                                        self.event_info = 'Checksum for object path: %s is %s and premis object checksum is %s. The checksum must match!' % (
                                            self.filepath, self.F_Checksum,
                                            self.object[2])
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create(
                                            '1025', '',
                                            'ESSArch SIPValidateFormat',
                                            ProcVersion, '1', self.event_info,
                                            self.DBmode,
                                            self.ObjectIdentifierValue)
                                        self.ok = 0
                                        break
                            elif self.ok and not self.ObjectNumItems == 0 and (
                                    self.object[5] == 'ARCHMETA'
                                    or self.object[5] == 'ARCHMETAxmlWrap'):
                                if int(os.stat(self.filepath)[6]) > 0:
                                    pass
                                else:
                                    self.event_info = 'Filesize for object path: %s is 0 bytes. The size should be more then 0 bytes!' % self.filepath
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1025', '',
                                        'ESSArch SIPValidateFormat',
                                        ProcVersion, '1', self.event_info,
                                        self.DBmode,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                        elif self.metatype in [4]:
                            if self.ok:
                                #[objectIdentifierValue,messageDigestAlgorithm,messageDigest,a_SIZE,a_MIMETYPE]
                                if int(os.stat(self.filepath)[6]) == int(
                                        self.object[3]):
                                    self.ObjectSize += int(self.object[3])
                                else:
                                    self.event_info = 'Filesize for object path: %s is %s and METS object size is %s. The sizes must match!' % (
                                        self.filepath,
                                        str(os.stat(self.filepath)[6]),
                                        str(self.object[3]))
                                    logging.error(self.event_info)
                                    ESSPGM.Events().create(
                                        '1025', '',
                                        'ESSArch SIPValidateFormat',
                                        ProcVersion, '1', self.event_info,
                                        self.DBmode,
                                        self.ObjectIdentifierValue)
                                    self.ok = 0
                                    break
                                if self.ok:
                                    self.F_Checksum, errno, why = ESSPGM.Check(
                                    ).checksum(self.filepath,
                                               self.object[1])  # Checksum
                                    if errno:
                                        self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
                                            self.filepath, str(why))
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create(
                                            '1025', '',
                                            'ESSArch SIPValidateFormat',
                                            ProcVersion, '1', self.event_info,
                                            self.DBmode,
                                            self.ObjectIdentifierValue)
                                        self.ok = 0
                                if self.ok:
                                    if self.F_Checksum == self.object[2]:
                                        pass
                                    else:
                                        self.event_info = 'Checksum for object path: %s is %s and METS object checksum is %s. The checksum must match!' % (
                                            self.filepath, self.F_Checksum,
                                            self.object[2])
                                        logging.error(self.event_info)
                                        ESSPGM.Events().create(
                                            '1025', '',
                                            'ESSArch SIPValidateFormat',
                                            ProcVersion, '1', self.event_info,
                                            self.DBmode,
                                            self.ObjectIdentifierValue)
                                        self.ok = 0
                                        break
                        self.ObjectNumItems += 1
                if self.ok:
                    if self.metatype in [1, 2, 3]:
                        ###############################################################################
                        # Check if SIP filesystem path contain files that not exist in metadatafile
                        for self.filesystem_object in ESSPGM.Check(
                        ).GetFiletree(
                                os.path.join(self.SIPpath,
                                             self.ObjectIdentifierValue)):
                            self.missmatch_flag = 0
                            for self.object in self.object_list:
                                #if os.path.join(self.ObjectIdentifierValue,self.filesystem_object) == self.object[0].encode('utf-8'):
                                if os.path.join(self.ObjectIdentifierValue,
                                                self.filesystem_object
                                                ) == self.object[0]:
                                    self.missmatch_flag = 0
                                    break
                                else:
                                    self.missmatch_flag = 1
                            if self.missmatch_flag:
                                self.filesystempath = u'%s/%s/%s' % (
                                    self.SIPpath, self.ObjectIdentifierValue,
                                    self.filesystem_object)
                                self.event_info = 'Filesystem file: %s do not exist in metadatafile for object: %s' % (
                                    self.filesystempath,
                                    os.path.join(self.SIPpath,
                                                 self.ObjectIdentifierValue))
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1025', '', 'ESSArch SIPValidateFormat',
                                    ProcVersion, '1', self.event_info,
                                    self.DBmode, self.ObjectIdentifierValue)
                                self.ok = 0
                    if self.metatype in [4]:
                        ###############################################################################
                        # Check if SIP filesystem path contain files that not exist in metadatafile
                        for self.filesystem_object in ESSPGM.Check(
                        ).GetFiletree(self.SIPpath):
                            self.missmatch_flag = 0
                            for self.object in self.object_list:
                                if self.filesystem_object == self.object[0]:
                                    self.missmatch_flag = 0
                                    break
                                else:
                                    self.missmatch_flag = 1
                            if self.missmatch_flag:
                                self.filesystempath = u'%s/%s' % (
                                    self.SIPpath, self.filesystem_object)
                                self.event_info = 'Filesystem file: %s do not exist in metadatafile for object: %s' % (
                                    self.filesystempath, self.SIPpath)
                                logging.error(self.event_info)
                                ESSPGM.Events().create(
                                    '1025', '', 'ESSArch SIPValidateFormat',
                                    ProcVersion, '1', self.event_info,
                                    self.DBmode, self.ObjectIdentifierValue)
                                self.ok = 0
                if self.ok:
                    if self.metatype in [4]:
                        ###############################################################################
                        # Check if SIP INFORMATIONCLASS match Policy
                        if self.INFORMATIONCLASS == self.Policy_INFORMATIONCLASS:
                            self.event_info = 'Object: %s InformationClass: %s match defined InformaionClass: %s in PolicyID: %s' % (
                                self.ObjectIdentifierValue,
                                self.INFORMATIONCLASS,
                                self.Policy_INFORMATIONCLASS, self.PolicyId)
                            logging.info(self.event_info)
                        else:
                            self.event_info = 'Object: %s InformationClass: %s do not match defined InformationClass: %s in PolicyID: %s' % (
                                self.ObjectIdentifierValue,
                                self.INFORMATIONCLASS,
                                self.Policy_INFORMATIONCLASS, self.PolicyId)
                            logging.error(self.event_info)
                            ESSPGM.Events().create(
                                '1025', '', 'ESSArch SIPValidateFormat',
                                ProcVersion, '1', self.event_info, self.DBmode,
                                self.ObjectIdentifierValue)
                            self.ok = 0
                if self.ok:
                    self.stopTime = datetime.timedelta(
                        seconds=time.localtime()[5],
                        minutes=time.localtime()[4],
                        hours=time.localtime()[3])
                    self.MeasureTime = self.stopTime - self.startTime
                    self.ObjectSizeMB = self.ObjectSize / 1048576
                    if self.MeasureTime.seconds < 1:
                        self.MeasureTime = datetime.timedelta(
                            seconds=1
                        )  #Fix min time to 1 second if it is zero.
                    self.VerMBperSEC = int(self.ObjectSizeMB) / int(
                        self.MeasureTime.seconds)
                if self.ok:
                    logging.info('Succeeded to validate SIP package: ' +
                                 self.ObjectIdentifierValue + ' , ' +
                                 str(self.VerMBperSEC) + ' MB/Sec and Time: ' +
                                 str(self.MeasureTime))
                    errno, why = ESSPGM.DB().SetAIPstatus(
                        self.IngestTable, self.ext_IngestTable,
                        AgentIdentifierValue, self.ObjectUUID, 29, 0)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    else:
                        ESSPGM.Events().create('1025', '',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '0', '',
                                               self.DBmode,
                                               self.ObjectIdentifierValue)
                else:
                    errno, why = ESSPGM.DB().SetAIPstatus(
                        self.IngestTable, self.ext_IngestTable,
                        AgentIdentifierValue, self.ObjectUUID, 26, 4)
                    if errno:
                        logging.error('Failed to update DB status for AIP: ' +
                                      str(self.ObjectIdentifierValue) +
                                      ' error: ' + str(why))
                    else:
                        self.event_info = 'Failed to validate SIP package: ' + self.ObjectIdentifierValue
                        logging.error(self.event_info)
                        ESSPGM.Events().create('1025', '',
                                               'ESSArch SIPValidateFormat',
                                               ProcVersion, '1',
                                               self.event_info, self.DBmode,
                                               self.ObjectIdentifierValue)
            db.close_old_connections()
            self.mLock.release()
            time.sleep(int(self.Time))
        self.mDieFlag = 0
Exemple #12
0
 def ThreadMain(self,ProcName):
     logging.info('Starting ' + ProcName)
     self.tz=timezone.get_default_timezone()
     while 1:
         if self.mDieFlag==1: break      # Request for death
         self.mLock.acquire()
         self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
         if self.Run == '0':
             logging.info('Stopping ' + ProcName)
             ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
             self.RunFlag=0
             self.mLock.release()
             #if Debug: print 'RunFlag: 0'
             time.sleep(2)
             continue
         # Process Item 
         lock=thread.allocate_lock()
         self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
         self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
         Cmets_obj = Parameter.objects.get(entity='content_descriptionfile').value
         if ExtDBupdate:
             self.ext_IngestTable = self.IngestTable
         else:
             self.ext_IngestTable = ''
         self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue','ObjectUUID','PolicyId'),('StatusProcess','BETWEEN',29,'AND',31,'AND',
                                                                                                                'StatusActivity','=','0'))
         if errno: logging.error('Failed to access Local DB, error: ' + str(why))
         for self.obj in self.dbget:
             self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
             if self.ProcDB[0]=='0':
                 logging.info('Stopping ' + ProcName)
                 ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
                 thread.interrupt_main()
                 time.sleep(5)
                 break
             elif self.ProcDB[1]==1:
                 while 1:
                     time.sleep(60)
                     self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
                     if self.ProcDB[1]==1:
                         logging.info('Process is in pause state')
                     else:
                         break
             self.ok = 1
             ###########################################################
             # get policy info
             self.ObjectIdentifierValue = self.obj[0]
             self.ObjectUUID = self.obj[1]
             self.PolicyId = self.obj[2]
             logging.info('Start to create AIP for: %s', self.ObjectIdentifierValue)
             self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','IngestPath'),('PolicyID',self.PolicyId))
             if errno: 
                 logging.error('Failed to access Local DB, error: ' + str(why))
                 self.ok = 0
             if self.ok:
                 ###########################################################
                 # set variables
                 self.AIPpath = self.PolicyDB[0][0]
                 self.metatype = self.PolicyDB[0][1]
                 self.SIPpath = self.PolicyDB[0][2]
                 self.p_obj = self.ObjectIdentifierValue + '.tar'
                 self.p_objpath = os.path.join(self.AIPpath,self.p_obj)
                 #self.Cmets_obj = self.ObjectIdentifierValue + '_Content_METS.xml'
                 #self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
                 self.Cmets_obj = Cmets_obj.replace('{objid}',self.ObjectIdentifierValue)
                 self.SIProotpath = os.path.join(self.SIPpath,self.ObjectIdentifierValue)
                 if self.metatype in [4]:
                     self.Cmets_objpath = os.path.join(self.SIProotpath,self.Cmets_obj)
                     if os.path.exists(os.path.join(self.SIProotpath,'sip.xml')):
                         mets_file = 'sip.xml'
                         self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
                     elif os.path.exists(os.path.join(self.SIProotpath,'mets.xml')):
                         mets_file = 'mets.xml'
                         self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
                     #elif os.path.exists(os.path.join(self.SIProotpath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
                     #    mets_file = '%s_Content_METS.xml' % self.ObjectIdentifierValue
                     #    self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
                     else:
                         self.SIPmets_objpath = ''
                 elif self.metatype in [1,2,3]:
                     self.Cmets_objpath = os.path.join(self.AIPpath,self.Cmets_obj)
                 Debug = 1
                 logging.debug('self.obj: %s', str(self.obj))
                 logging.debug('self.ObjectIdentifierValue: %s', self.ObjectIdentifierValue)
                 logging.debug('Len self.ObjectIdentifierValue: %s', len(self.ObjectIdentifierValue))
                 logging.debug('self.SIPpath: %s', self.SIPpath)
                 logging.debug('self.AIPpath: %s', self.AIPpath)
             if self.metatype in [1,2,3]:
                 if self.ok:
                     ###########################################################
                     # get object_list from PREMIS file 
                     self.Premis_filepath = '%s/%s/%s_PREMIS.xml' % (self.SIPpath,self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                     self.object_list,errno,why = ESSMD.getPremisObjects(FILENAME=self.Premis_filepath)
                     # list [objectIdentifierValue,messageDigestAlgorithm,messageDigest,messageDigestOriginator,size,formatName,formatVersion]
                     if errno == 0:
                         logging.info('Succeeded to get object_list from premis for information package: %s', self.ObjectIdentifierValue)
                     else:
                         self.event_info = 'Problem to get object_list from premis for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                         logging.error(self.event_info)
                         ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                         self.ok = 0
                 if self.ok:
                     ###########################################################
                     # create AIP content METS file
                     #self.firstPremisObjectFlag = 1
                     METS_agent_list = []
                     if self.metatype == 1:
                         ############################################
                         # Object have metatype 1 (METS)
                         self.METS_LABEL = 'ESSArch AIP'
                         self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                         self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
                         self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
                         # Get SIP Content METS information
                         self.METSfilepath = os.path.join(self.SIPpath,self.ObjectIdentifierValue + '/metadata/SIP/' + self.ObjectIdentifierValue + '_Content_METS.xml') 
                         res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.METSfilepath)
                         for agent in res_info[2]:
                             if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
                                 METS_agent_list.append(agent)
                         METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                         METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                     elif self.metatype == 2:
                         ############################################
                         # Object have metatype 2 (RES)
                         self.METS_LABEL = 'Imaging AIP RA'
                         self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                         self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
                         self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
                         METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
                         METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
                         METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                         METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                     elif self.metatype == 3:
                         ############################################
                         # Object have metatype 3 (ADDML)
                         self.METS_LABEL = 'Born Digital AIP RA'
                         self.tmp_object_id = ('%s/%s_ADDML.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                         self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
                         self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'ADDML'])
                         self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
                         self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
                         self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
                         METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
                         METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
                         METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
                         METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
                     self.firstPremisObjectFlag = 1
                     self.DataObjectNumItems = 0
                     self.DataObjectSize = 0
                     self.MetaObjectSize = 0
                     self.MetaObjectIdentifier = 'None'
                     for self.object in self.object_list:
                         self.filepath = os.path.join(self.SIPpath, self.object[0])
                         self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
                         self.a_filepath = self.object[0]
                         if self.firstPremisObjectFlag:
                             if self.object[0] == self.ObjectIdentifierValue:
                                 logging.info('First premis object match information package: %s', self.ObjectIdentifierValue)
                                 if self.metatype == 1:
                                     self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis'])
                                 elif self.metatype == 2:
                                     self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis','mix'])
                                 elif self.metatype == 3:
                                     self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis','addml','xhtml'])
                                 self.firstPremisObjectFlag = 0
                                 continue
                             else:
                                 self.event_info = 'First premis object do not match information package: %s, premis_object: %s' % (self.ObjectIdentifierValue,self.object[0])
                                 logging.error(self.event_info)
                                 ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                 self.ok = 0
                         elif os.access(self.filepath_iso,os.R_OK):
                             self.file_statinfo = os.stat(self.filepath_iso)
                             if self.metatype == 2 and self.object[5] == 'ARCHMETA':
                                 ############################################
                                 # Object have metatype 2 and RES file
                                 #self.file_ID = string.replace(self.object[0],'/','%')
                                 self.file_ID = self.object[0]
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'Content description'
                                 self.file_MIMETYPE = 'text/csv'
                                 self.file_MDTYPE = 'OTHER'
                                 self.file_OTHERMDTYPE = 'RES'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                             elif self.object[5] == 'ARCHMETAxmlWrap' and self.object[6] == 'PREMIS':
                                 ############################################
                                 # Object is a PREMIS XML file
                                 self.file_MDTYPE = 'PREMIS'
                                 self.file_OTHERMDTYPE = ''
                             elif self.object[5] == 'ARCHMETAxmlWrap':
                                 ############################################
                                 # Object is a OTHER XML file
                                 self.file_MDTYPE = 'OTHER'
                                 self.file_OTHERMDTYPE = self.object[6]
                             elif self.metatype == 1:
                                 ############################################
                                 # Object have metatype 1, convert PREMIS formatName to MIME-type, datafile
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'Datafiles'
                                 self.file_MIMETYPE = ESSPGM.Check().PREMISformat2MIMEtype(self.object[5])
                                 self.file_USE = 'Datafile'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                                 if self.file_MIMETYPE == 'unknown':
                                     self.event_info = 'Problem to idetify MIMETYPE from PREMIS for: %s' % self.filepath
                                     logging.error(self.event_info)
                                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                     self.ok = 0
                             elif self.metatype == 2 and self.object[0][-12:] == 'TIFFEdit.RES':
                                 ############################################
                                 # Object have metatype 2 and RES file
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'RA Information'
                                 self.file_MIMETYPE = 'text/csv'
                                 self.file_USE = 'RA Information'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                             elif self.metatype == 2:
                                 ############################################
                                 # Object have metatype 2 and datafile is an tiff image
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'RA Datafiles'
                                 self.file_MIMETYPE = 'image/tiff'
                                 self.file_USE = 'RA Datafile'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                             elif self.metatype == 3:
                                 ############################################
                                 # Object have metatype 3, convert PREMIS formatName to MIME-type, datafile 
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'Datafiles'
                                 self.file_MIMETYPE = ESSPGM.Check().PREMISformat2MIMEtype(self.object[5])
                                 self.file_USE = 'Datafile'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                                 if self.file_MIMETYPE == 'unknown':
                                     self.event_info = 'Problem to idetify MIMETYPE from PREMIS for: %s' % self.filepath
                                     logging.error(self.event_info)
                                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                     self.ok = 0
                             else:
                                 ############################################
                                 # Object is a datafile
                                 self.file_ID = self.object[0]
                                 self.file_SIZE = self.file_statinfo.st_size
                                 self.file_LABEL = 'Datafiles'
                                 self.file_MIMETYPE = 'xxxxx'                # Maste fixas
                                 self.file_USE = 'Datafile'
                                 self.file_CHECKSUMTYPE = self.object[1]
                                 self.file_CHECKSUM = self.object[2]
                                 self.file_LOCTYPE = 'URL'
                                 self.file_xlink_type = 'simple'
                         else:
                             self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
                             logging.error(self.event_info)
                             ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                             self.ok = 0
                         if self.ok:
                             ###########################################################
                             # add files to METS file
                             if not (self.object[5] == 'ARCHMETA' or self.object[5] == 'ARCHMETAxmlWrap'):
                                 self.DataObjectNumItems += 1
                                 self.DataObjectSize += self.file_SIZE
                                 self.fil_utc_mtime = datetime.datetime.utcfromtimestamp(self.file_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                                 self.fil_lociso_mtime = self.fil_utc_mtime.astimezone(self.tz).isoformat()
                                 self.METSdoc = ESSMD.AddDataFiles(self.METSdoc,self.file_LABEL,'FILES','',[(self.file_ID,self.file_SIZE,self.fil_lociso_mtime,self.file_MIMETYPE,'',self.file_USE,self.file_CHECKSUMTYPE,self.file_CHECKSUM,self.file_LOCTYPE,self.file_xlink_type)])
                             elif self.object[5] == 'ARCHMETA':
                                 self.MetaObjectSize += self.file_SIZE
                                 self.fil_utc_mtime = datetime.datetime.utcfromtimestamp(self.file_statinfo.st_mtime).replace(tzinfo=pytz.utc)
                                 self.fil_lociso_mtime = self.fil_utc_mtime.astimezone(self.tz).isoformat()
                                 self.METSdoc = ESSMD.AddContentFiles(self.METSdoc,self.file_LABEL,'',[(self.file_ID,self.file_SIZE,self.fil_lociso_mtime,self.file_MIMETYPE,self.file_MDTYPE,self.file_OTHERMDTYPE,self.file_CHECKSUMTYPE,self.file_CHECKSUM,self.file_LOCTYPE,self.file_xlink_type)])
                             elif self.object[5] == 'ARCHMETAxmlWrap':
                                 logging.info('Wrap XML file: ' + self.a_filepath + ' to METS file')
                                 self.file_xml,errno,why = ESSMD.parseFromFile(self.filepath)
                                 if errno:
                                     self.event_info = 'Failed to parse XML file: ' + str(self.filepath) + ' error: ' + str(why)
                                     logging.error(self.event_info)
                                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                     self.ok = 0
                                 self.METSdoc = ESSMD.AddContentEtree(self.METSdoc,[(self.file_xml,self.file_MDTYPE,self.file_OTHERMDTYPE)])
                 if self.ok:
                     ########################
                     # Update root schemalocation and remove all other schemalocation
                     self.METSdoc,errno,why = ESSMD.updateSchemaLocation(self.METSdoc)
                     ########################
                     # Update all ADMID in DOC
                     res,errno,why = ESSMD.updateFilesADMID(self.METSdoc)
                     ########################
                     # Set xml_METS to self.METSdoc
                     xml_METS = self.METSdoc
             if self.metatype in [4]:
                 dt = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                 loc_dt_isoformat = dt.astimezone(self.tz).isoformat()
                 xml_METS = ESSMD.updatePackage(FILENAME=self.SIPmets_objpath,TYPE='AIP',CREATED=loc_dt_isoformat,metsDocumentID=self.Cmets_obj)
             if self.ok:
                 ########################
                 # Write METS file
                 errno,why = ESSMD.writeToFile(xml_METS,self.Cmets_objpath)
                 if errno:
                     self.event_info = 'Problem to write METS to file for AIP package: ' + str(self.p_objpath)
                     logging.error(self.event_info)
                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                     time.sleep(2)
                     self.ok = 0
                 self.MetaObjectIdentifier = self.Cmets_obj
                 self.MetaObjectSize = 0
             if self.ok:
                 ###########################################################
                 # get object_list from METS 
                 self.object_list,errno,why = ESSMD.getAIPObjects(FILENAME=self.Cmets_objpath)
                 if errno == 0:
                     logging.info('Succeeded to get object_list from METS for information package: %s', self.ObjectIdentifierValue)
                 else:
                     self.event_info = 'Problem to get object_list from METS for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
                     logging.error(self.event_info)
                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                     self.ok = 0
             if self.ok:
                 ###########################################################
                 # Insert METS file as first object in AIP package
                 self.tmp_object_size = os.stat(self.Cmets_objpath)[6]
                 self.object_list.insert(0,[self.Cmets_obj,'','',self.tmp_object_size,''])
             if self.ok:
                 ###########################################################
                 # create AIP package file
                 try:
                     errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 30, 5)
                     if errno: logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                     logging.info('Create AIP Package: ' + self.p_objpath)
                     self.tarfile = tarfile.open(self.p_objpath, "w",)
                 except tarfile.TarError:
                     self.event_info = 'Problem to create AIP Package: ' + str(self.p_objpath)
                     logging.error(self.event_info)
                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                     self.ok = 0
             if self.ok:
                 ###########################################################
                 # add files to AIP package file
                 self.startTarTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                 self.firstPremisObjectFlag = 1
                 self.ObjectNumItems = 0
                 self.ObjectSize = 0
                 for self.object in self.object_list: 
                     if self.metatype in [1,2,3]:
                         self.a_filepath = self.object[0]
                     elif self.metatype in [4]:
                         self.a_filepath = '%s/%s' % (self.ObjectIdentifierValue,self.object[0])
                     self.a_filepath_iso = ESSPGM.Check().unicode2str(self.a_filepath)
                     self.object_size = int(self.object[3])
                     if self.a_filepath == self.Cmets_obj or self.a_filepath == '%s/%s' % (self.ObjectIdentifierValue,self.Cmets_obj):
                         self.filepath = self.Cmets_objpath
                         if self.metatype in [4]:
                             self.MetaObjectSize = self.object_size
                             self.DataObjectSize = 0
                             self.DataObjectNumItems = 0
                     else:
                         self.filepath = os.path.join(self.SIPpath, self.a_filepath)
                         if self.metatype in [4]:
                             self.DataObjectSize += self.object_size
                             self.DataObjectNumItems += 1
                     self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
                     if os.access(self.filepath_iso,os.R_OK):
                         if int(os.stat(self.filepath_iso)[6]) == self.object_size:
                             try:
                                 self.ObjectNumItems += 1
                                 self.tarinfo = self.tarfile.gettarinfo(self.filepath_iso, self.a_filepath_iso)
                                 self.tarfile.addfile(self.tarinfo, file(self.filepath_iso))
                                 logging.info('Add: ' + self.a_filepath + ' to AIP Package: ' + self.p_obj)
                             except tarfile.TarError:
                                 self.event_info = 'Problem to add: ' + str(self.a_filepath) + ' to AIP Package: ' + str(self.p_objpath)
                                 logging.error(self.event_info)
                                 ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                                 self.ok = 0
                         else:
                             self.event_info = 'Filesize for object path: %s is %s and METS object size is %s. The sizes must match!' % (self.filepath,str(os.stat(self.filepath_iso)[6]),str(self.object_size))
                             logging.error(self.event_info)
                             ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                             self.ok = 0
                     else:
                         self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
                         logging.error(self.event_info)
                         ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                         self.ok = 0
             if self.ok:
                 ###########################################################
                 # Close AIP package
                 try: 
                     self.tarfile.close()
                 except tarfile.TarError: 
                     self.event_info = 'Problem to close AIP package: ' + str(self.p_objpath)
                     logging.error(self.event_info)
                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
                     self.ok = 0
             if self.ok:
                 ###########################################################
                 # Check if StatusActivity is OK
                 self.ObjectSize = os.stat(self.p_objpath)[6]
                 self.stopTarTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
                 self.TarTime = self.stopTarTime-self.startTarTime
                 self.WriteSize = int(self.ObjectSize)/1048576 
                 if self.TarTime.seconds < 1: self.TarTime = datetime.timedelta(seconds=1)	#Fix min time to 1 second if it is zero.
                 self.TarMBperSEC = int(self.WriteSize)/int(self.TarTime.seconds)
                 logging.info('Close AIP package: ' + self.p_obj)
                 logging.info('Succeeded to create AIP for: ' + self.ObjectIdentifierValue + ' , ' + str(self.TarMBperSEC) + ' MB/Sec and Time: ' + str(self.TarTime))
                 self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                 self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                 res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectPackageName',self.p_obj,
                                                                           'ObjectSize',self.ObjectSize,
                                                                           'ObjectNumItems',self.ObjectNumItems,
                                                                           'ObjectMessageDigest','',
                                                                           'ObjectPath','',
                                                                           'MetaObjectIdentifier',self.MetaObjectIdentifier,
                                                                           'MetaObjectSize',self.MetaObjectSize,
                                                                           'DataObjectSize',self.DataObjectSize,
                                                                           'DataObjectNumItems',self.DataObjectNumItems,
                                                                           'CreateDate',self.timestamp_utc.replace(tzinfo=None),
                                                                           'CreateAgentIdentifierValue',AgentIdentifierValue,
                                                                           'StatusProcess','39',
                                                                           'StatusActivity','0',
                                                                           'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                           'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                           'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                          ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                 if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                 else: ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'0','',2,self.ObjectIdentifierValue)
                 if errno == 0 and self.ext_IngestTable:
                     ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.ext_IngestTable,'UPD',('ObjectPackageName',self.p_obj,
                                                                                              'ObjectSize',self.ObjectSize,
                                                                                              'ObjectNumItems',self.ObjectNumItems,
                                                                                              'ObjectMessageDigest','',
                                                                                              'ObjectPath','',
                                                                                              'MetaObjectIdentifier',self.MetaObjectIdentifier,
                                                                                              'MetaObjectSize',self.MetaObjectSize,
                                                                                              'DataObjectSize',self.DataObjectSize,
                                                                                              'DataObjectNumItems',self.DataObjectNumItems,
                                                                                              'CreateDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                              'CreateAgentIdentifierValue',AgentIdentifierValue,
                                                                                              'StatusProcess','39',
                                                                                              'StatusActivity','0',
                                                                                              'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                              'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                             ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                     if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                     else:
                         res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                         if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
             else:
                 errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 31, 4)
                 if errno: 
                     logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                 else:
                     self.event_info = 'Failed to create AIP package: ' + self.p_obj
                     logging.error(self.event_info)
                     ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
         db.close_old_connections()
         self.mLock.release()
         time.sleep(int(self.Time))
     self.mDieFlag=0
Exemple #13
0
    def ObjectValidate(self):
        self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0] 
        self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0] 
        if ExtDBupdate:
            self.ext_IngestTable = self.IngestTable
        else:
            self.ext_IngestTable = ''

        # Check if exist extDB and got projektid
        self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue','PolicyID','StatusProcess','StatusActivity'),('StatusActivity','=','0','AND',
                                                                                                               'StatusProcess','BETWEEN',9,'AND',14,
                                                                                                               'OR',
                                                                                                               'StatusActivity','=','4','AND',
                                                                                                               'StatusProcess','BETWEEN',10,'AND',11))
        if errno: logging.error('Failed to access Local DB, error: ' + str(why))
        for self.obj in self.dbget:
            self.ObjectIdentifierValue = self.obj[0]
            self.PolicyID = self.obj[1]
            self.StatusProcess = self.obj[2]
            self.StatusActivity = self.obj[3]
            self.DBmode = 0
            self.ext_ObjectGuid = None
            self.ext_EntryDate = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
            self.ext_EntryAgentIdentifierValue = None
            self.ext_OAISPackageType = 2
            self.ext_preservationLevelValue = 1
            self.ext_ObjectActive = 0
            self.objectstatus = 0
            self.ext_ProjectGroupCode = None
            self.ext_ObjectPackageName = ''
            if Debug: logging.info('StatusProcess 9, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
            #Check....
            if self.PolicyID:
                #q_ESSArchPolicy = model.meta.Session.query(model.ESSArchPolicy)
                #DbRow = q_ESSArchPolicy.filter(and_(model.ESSArchPolicy.PolicyStat==1, \
                #                                    model.ESSArchPolicy.PolicyID==int(self.PolicyID))).first()
                DbRow = ESSArchPolicy.objects.filter( PolicyStat = 1, PolicyID = int(self.PolicyID) )[:1]
                if DbRow:
                    DbRow = DbRow.get()
                    if DbRow.Mode in range(0,2):
                        self.DBmode = DbRow.Mode
                        logging.info('Policy found for Object: %s in ESSArch mode' % self.ObjectIdentifierValue)
                        if DbRow.IngestMetadata in [1,2,3]:
                            metsfilename = os.path.join(DbRow.IngestPath,self.ObjectIdentifierValue + '_Package_METS.xml')
                        elif DbRow.IngestMetadata in [4]:
                            ObjectPath = os.path.join(DbRow.IngestPath,self.ObjectIdentifierValue)
                            if os.path.exists(os.path.join(ObjectPath,'sip.xml')):
                                metsfilename = os.path.join(ObjectPath,'sip.xml')
                            elif os.path.exists(os.path.join(ObjectPath,'mets.xml')):
                                metsfilename = os.path.join(ObjectPath,'mets.xml')
                            #elif os.path.exists(os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
                            #    metsfilename = os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)
                            else:
                                metsfilename = ''
                            #metsfilename = '%s/sip.xml' % os.path.join(DbRow.IngestPath,self.ObjectIdentifierValue)
                        else:
                            metsfilename = ''
                        res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=metsfilename)
                        if not error:
                            # cut off microsecond and timezone info ".xxxxxxx+02:00" 
                            #for c in res_info[1][0]:
                            #    if c == '.' or c == '+':
                            #        break
                            #    else:
                            #        self.ext_EntryDate += c
                            self.ext_EntryDate = parse_datetime(res_info[1][0]).astimezone(pytz.utc)
                            self.ext_EntryAgentIdentifierValue = res_info[2][0][4]
                            try:
                                self.ext_ObjectGuid = str(uuid.UUID(self.ObjectIdentifierValue))
                            except ValueError, why:
                                logging.warning('ObjectIdentifierValue: %s is not a valid UUID, why: %s , start to generate a new UUID' % (self.ObjectIdentifierValue, str(why)))
                                self.ext_ObjectGuid = str(uuid.uuid1())
                                logging.info('New UUID: %s for ObjectIdentifierValue: %s' % (self.ext_ObjectGuid,str(self.ObjectIdentifierValue)))
                            self.ext_ObjectActive = 1
                            self.ext_OAISPackageType = 2
                            self.ext_preservationLevelValue = 1
                            self.objectstatus = 1
                        else:
                            self.objectstatus = 102 # Problem to get information from package METS 
                    elif DbRow.Mode == 2: # AIS but POLICYID from METS, Check in AIS if object is active.
                        self.DBmode = DbRow.Mode
                        self.extOBJdbget,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'GET3',('ProjectGroupCode',
                                                                                                           'ObjectPackageName',
                                                                                                           'ObjectGuid',
                                                                                                           'ObjectActive',
                                                                                                           'EntryDate',
                                                                                                           'EntryAgentIdentifierValue',
                                                                                                           'OAISPackageType',
                                                                                                           'preservationLevelValue'),
                                                                                                          ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                        #self.extOBJdbget = [[10,'','7283074a-00c0-11e2-a78f-002215836500',1,'2010-07-12 16:57:45','entryagent',2,1]]
                        #ext_errno = 0
                        #ext_why = 'whywhy'
                        if ext_errno: logging.error('Failed to access External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                        elif self.extOBJdbget:
                            if Debug: logging.info('Found object: %s in AIS, self.extOBJdbget: %s',self.ObjectIdentifierValue,str(self.extOBJdbget))
                            self.objectstatus = 10 # Object found in external DB
                            if self.objectstatus < 100:
                                ########################################
                                # Check if object alredy have an AIP
                                if disable_ObjectPackageName:
                                    self.ext_ObjectPackageName = ''
                                else:
                                    self.ext_ObjectPackageName = self.extOBJdbget[0][1]
                                if not self.ext_ObjectPackageName:
                                    self.objectstatus = 12 # Object do not have an AIP
                                else:
                                    self.objectstatus = 112 # Object already have an AIP
                            if self.objectstatus < 100:
                                ########################################
                                # Get GUID/UUID
                                #self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2])   #When pymssql
                                self.ext_ObjectGuid = uuid.UUID(self.extOBJdbget[0][2])
                            if self.objectstatus < 100:
                                ########################################
                                # Check if object is active
                                self.ext_ObjectActive = self.extOBJdbget[0][3]
                                if self.ext_ObjectActive == 1:
                                    self.objectstatus = 13 # Object is active
                                else:
                                    self.objectstatus = 113 # Object is not active
                            if self.objectstatus < 100:
                                ########################################
                                # Check if POLICYID in local DB "METS" is equal to ProjectGroupCode in AIS
                                self.ext_ProjectGroupCode = self.extOBJdbget[0][0]
                                if self.ext_ProjectGroupCode == self.PolicyID:
                                    self.objectstatus = 1 # Object have an ProjectCode
                                    logging.info('Object: %s found in AIS with correct POLICYID' % self.ObjectIdentifierValue)
                                else:
                                    self.objectstatus = 111 # Object do not have an ProjectCode
                            ########################################
                            self.ext_EntryDate = self.extOBJdbget[0][4].replace(microsecond=0,tzinfo=self.tz).astimezone(pytz.utc)
                            self.ext_EntryAgentIdentifierValue = self.extOBJdbget[0][5]
                            self.ext_OAISPackageType = self.extOBJdbget[0][6]
                            self.ext_preservationLevelValue = self.extOBJdbget[0][7]
                            ########################################
                            # Special function only for test
                            if force_ProjectGroupCode:
                                logging.info('Force set ProjectGroupCode for Object: %s' % self.ObjectIdentifierValue)
                                self.objectstatus = 10
                                self.ext_ProjectGroupCode = force_ProjectGroupCode
                                self.ext_ObjectPackageName = ''
                                self.ext_ObjectGuid = str(uuid.uuid1()) # updDB
                                self.ext_ObjectActive = 1 # updDB
                                self.ext_EntryDate = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc) # updDB
                                self.ext_EntryAgentIdentifierValue = None # updDB
                                self.ext_OAISPackageType = 2 # updDB
                                self.ext_preservationLevelValue = 1 # updDB
                        else:
                            self.objectstatus = 110 # Object not found in external DB
                            if Debug: logging.info('Missing object: %s in AIS, self.extOBJdbget: %s',self.ObjectIdentifierValue,str(self.extOBJdbget))
                    else:
                        self.objectstatus = 100 # Policy is not in ESSArch mode 
                else:
                    self.objectstatus = 101 # Policy not found or not active 
                #model.meta.Session.close()
            else: 
                self.DBmode = 2 # AIS 
                self.extOBJdbget,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'GET3',('ProjectGroupCode',
                                                                                                   'ObjectPackageName',
                                                                                                   'ObjectGuid',
                                                                                                   'ObjectActive',
                                                                                                   'EntryDate',
                                                                                                   'EntryAgentIdentifierValue',
                                                                                                   'OAISPackageType',
                                                                                                   'preservationLevelValue'),
                                                                                                  ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                if ext_errno: logging.error('Failed to access External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                elif self.extOBJdbget:
                #if not ext_errno and self.extOBJdbget:
                    if Debug: logging.info('Found object: %s in AIS, self.extOBJdbget: %s',self.ObjectIdentifierValue,str(self.extOBJdbget))
                    self.objectstatus = 10 # Object found in external DB
                    ########################################
                    # Check if object alredy have an AIP
                    self.ext_ProjectGroupCode = self.extOBJdbget[0][0]
                    if self.objectstatus < 100 and self.ext_ProjectGroupCode:
                        self.objectstatus = 11 # Object have an ProjectCode
                    else:
                        self.objectstatus = 111 # Object do not have an ProjectCode
                    ########################################
                    # Check if object alredy have an AIP
                    if disable_ObjectPackageName:
                        self.ext_ObjectPackageName = ''
                    else:
                        self.ext_ObjectPackageName = self.extOBJdbget[0][1]
                    if self.objectstatus < 100 and not self.ext_ObjectPackageName: 
                        self.objectstatus = 12 # Object do not have an AIP
                    else:
                        self.objectstatus = 112 # Object already have an AIP
                    ########################################
                    # Get GUID/UUID
                    #self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2])	#When pymssql
                    self.ext_ObjectGuid = uuid.UUID(self.extOBJdbget[0][2])
                    ########################################
                    # Check if object is active
                    self.ext_ObjectActive = self.extOBJdbget[0][3]
                    if self.objectstatus < 100 and self.ext_ObjectActive == 1:
                        self.objectstatus = 13 # Object is active
                    else:
                        self.objectstatus = 113 # Object is not active

                    ########################################
                    self.ext_EntryDate = self.extOBJdbget[0][4].replace(microsecond=0,tzinfo=self.tz).astimezone(pytz.utc)
                    self.ext_EntryAgentIdentifierValue = self.extOBJdbget[0][5]
                    self.ext_OAISPackageType = self.extOBJdbget[0][6]
                    self.ext_preservationLevelValue = self.extOBJdbget[0][7]
                    if Debug: logging.info('ext_ProjectGroupCode is: '+str(self.ext_ProjectGroupCode)+' for ObjectIdentifierValue '+str(self.ObjectIdentifierValue))
                    if force_ProjectGroupCode:
                        logging.info('Force set ProjectGroupCode for Object: %s' % self.ObjectIdentifierValue)
                        self.objectstatus = 10
                        self.ext_ProjectGroupCode = force_ProjectGroupCode
                        self.ext_ObjectPackageName = ''
                        self.ext_ObjectGuid = str(uuid.uuid1()) # updDB
                        self.ext_ObjectActive = 1 # updDB
                        self.ext_EntryDate = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc) # updDB
                        self.ext_EntryAgentIdentifierValue = None # updDB
                        self.ext_OAISPackageType = 2 # updDB
                        self.ext_preservationLevelValue = 1 # updDB
                    if self.objectstatus < 100:
                        self.PolicyID_dbget,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('PolicyID',),('AISProjectID',self.ext_ProjectGroupCode,'AND','PolicyStat',1))
                        if errno: 
                            logging.error('Failed to access Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                        elif self.PolicyID_dbget:
                            self.PolicyID = self.PolicyID_dbget[0][0]
                            if Debug: logging.info('PolicyID: '+str(self.PolicyID))
                            self.objectstatus = 14 # Object got PolicyID
                        else:
                            self.objectstatus = 114 # Object mising PolicyID

                else:
                    self.objectstatus = 110 # Object not found in external DB
                    if Debug: logging.info('Missing object: %s in AIS, self.extOBJdbget: %s',self.ObjectIdentifierValue,str(self.extOBJdbget))

            if self.objectstatus == 100: # Policy is not in ESSArch mode
                ###################################################################
                # Policy is not in ESSArch mode(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 12, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Policy is not in ESSArch mode for Object: ' + str(self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 101: # Policy not found or not active
                ###################################################################
                # Policy not found or active(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 12, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Policy not found or active for Object: ' + str(self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 102: # Problem to get information from Package_METS
                ###################################################################
                # Problem to get information from Package_METS(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 12, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Problem to get information from Package_METS for Object: ' + str(self.ObjectIdentifierValue)
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 110:
                ###################################################################
                #Object don't exist in extDB(10) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 10
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 10, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))

            elif self.objectstatus == 111:
                ###################################################################
                #Object don't have any projektkod in extDB(11) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 11
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 11, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))

            elif self.objectstatus == 112:
                ###################################################################
                #Object already have an AIP!!
                ###################################################################
                self.StatusProcess = 13
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 13, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(self.ObjectIdentifierValue) + ' already have an AIP!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 113:
                ###################################################################
                # Object is not active!!
                ###################################################################
                self.StatusProcess = 14
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 14, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(self.ObjectIdentifierValue) + ' is not active in external DB!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 114: 
                ###################################################################
                #Object don't have any local policy(12) and Need of assistance(4)
                ###################################################################
                self.StatusProcess = 12
                self.StatusActivity = 4
                if Debug: logging.info('Change to StatusProcess 12, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                self.event_info = 'Object: ' + str(self.ObjectIdentifierValue) + ' do not have any local policy!'
                logging.error(self.event_info)
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'1',self.event_info,self.DBmode,self.ObjectIdentifierValue)

            elif self.objectstatus == 14 or self.objectstatus == 1: 
                ###################################################################
                #Object got a policy(19) and RFNext and OK(0)
                ###################################################################
                self.StatusProcess = 19
                self.StatusActivity = 0
                if Debug: logging.info('Change to StatusProcess 19, ObjectIdentifierValue ' +str(self.ObjectIdentifierValue))
                ESSPGM.Events().create('1010','','ESSArch SIPValidateAIS',ProcVersion,'0','',self.DBmode,self.ObjectIdentifierValue)
        
            logging.info('objectstatus:%s,StatusProcess:%s,StatusActivity:%s,EntryDate:%s' % (self.objectstatus,self.StatusProcess,self.StatusActivity,self.ext_EntryDate))

            if self.objectstatus:
                self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
                self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
                res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('PolicyId',self.PolicyID,
                                                                          'ObjectUUID',self.ext_ObjectGuid,
                                                                          'EntryDate',self.ext_EntryDate.replace(tzinfo=None),
                                                                          'EntryAgentIdentifierValue',self.ext_EntryAgentIdentifierValue,
                                                                          'StatusProcess',self.StatusProcess,
                                                                          'StatusActivity',self.StatusActivity,
                                                                          'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
                                                                          'linkingAgentIdentifierValue',AgentIdentifierValue,
                                                                          'OAISPackageType',self.ext_OAISPackageType,
                                                                          'preservationLevelValue',self.ext_preservationLevelValue,
                                                                          'ObjectActive',self.ext_ObjectActive,
                                                                          'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
                                                                         ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
                if errno == 0 and self.ext_IngestTable:
                    ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('PolicyId',self.PolicyID,
                                                                                             'StatusProcess',self.StatusProcess,
                                                                                             'StatusActivity',self.StatusActivity,
                                                                                             'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
                                                                                             'linkingAgentIdentifierValue',AgentIdentifierValue),
                                                                                            ('ObjectIdentifierValue',self.ObjectIdentifierValue))
                    if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
                    else:
                        res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
                        if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))