def blob2text(self):
#self.table = 'IngestObjectMetadata'
res, errno, why = ESSDB.DB().action(
'IngestObjectMetadata', 'GET3',
('ObjectIdentifierValue', 'ObjectMetadataType',
'ObjectMetadataBLOB'), ('id', 9))
if errno:
print 'why', why
print res[0][2]
file = open('/ESSArch/bin/src/testdata/X0000003_MySQL.res', 'wb')
file.write(res[0][2])
file.close()
#self.table = 'IngestObjectMetadata'
res, errno, why = ESSMSSQL.DB().action(
'IngestObjectMetadata', 'GET3',
('ObjectIdentifierValue', 'ObjectMetadataType',
'ObjectMetadataBLOB'), ('ObjectIdentifierValue', 'X0000003',
'AND', 'ObjectMetadataType', 28))
if errno:
print 'why', why
for i in res:
print i[2]
file = open('/ESSArch/bin/src/testdata/X0000003_MsSQL.res', 'wb')
file.write(res[0][2])
file.close()
def projekttest_old(self, DataObjectIdentifier):
self.DataObjectIdentifier = DataObjectIdentifier
self.Packagedbget, ext_errno, ext_why = ESSMSSQL.DB().action(
'ArchivePackageDB', 'GET3', ('projektkod_fk', 'a_obj'),
('a_obj', self.DataObjectIdentifier))
if ext_errno:
print 'ext_why: %s' % str(ext_why)
if self.Packagedbget:
self.Projektdbget = ESSMSSQL.DB().action(
'ArchiveProjektkodDB', 'GET',
('Projektgrupp_fk', 'Projektgrupp_namn'),
('projektkod_id', self.Packagedbget[0][0]))
self.extPrjKod = self.Projektdbget[0][0]
self.extPrjNamn = self.Projektdbget[0][1]
print 'PrjKod: ' + str(self.extPrjKod) + ' PrjNamn: ' + str(
self.extPrjNamn)
def projekttest(self, ObjectIdentifierValue):
Debug = 1
self.ObjectIdentifierValue = ObjectIdentifierValue
self.IngestTable = 'IngestObject'
# self.extOBJdbget,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'GET3',('ProjectGroupCode',
# 'ObjectPackageName',
# 'ObjectGuid',
# 'ObjectActive',
# 'EntryDate',
# 'EntryAgentIdentifierValue'),
# ('ObjectIdentifierValue',self.ObjectIdentifierValue))
self.extOBJdbget, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'GET3',
('ProjectGroupCode', 'ObjectPackageName', 'ObjectGuid',
'ObjectActive', 'EntryDate', 'EntryAgentIdentifierValue',
'OAISPackageType', 'preservationLevelValue', 'ProjectGroupName'),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if ext_errno:
print('Failed to access External DB: ' +
str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
if not ext_errno and self.extOBJdbget:
#if Debug: print ('self.extOBJdbget: '+str(self.extOBJdbget)+' for ObjectIdentifierValue '+str(self.ObjectIdentifierValue))
self.ext_ProjectGroupCode = self.extOBJdbget[0][0]
self.ext_ObjectPackageName = self.extOBJdbget[0][1]
#self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2]) #When pymssql
self.ext_ObjectGuid = uuid.UUID(self.extOBJdbget[0][2])
self.ext_ObjectActive = self.extOBJdbget[0][3]
self.ext_EntryDate = self.extOBJdbget[0][4]
self.ext_EntryAgentIdentifierValue = self.extOBJdbget[0][5]
self.ext_OAISPackageType = self.extOBJdbget[0][6]
self.ext_preservationLevelValue = self.extOBJdbget[0][7]
self.ext_ProjectGroupName = self.extOBJdbget[0][8]
if Debug:
print('ext_ProjectGroupCode is: ' +
str(self.ext_ProjectGroupCode) +
' for ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
if Debug:
print('ext_ObjectActive is: ' + str(self.ext_ObjectActive) +
' for ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
return (self.ext_ProjectGroupCode, self.ext_ObjectPackageName,
self.ext_ObjectGuid, self.ext_ObjectActive,
self.ext_EntryDate, self.ext_EntryAgentIdentifierValue,
self.ext_OAISPackageType, self.ext_preservationLevelValue,
self.ext_ProjectGroupName), 0, self.ObjectIdentifierValue
return ('', '', '', '', '',
''), 1, '%s not found in ais' % self.ObjectIdentifierValue
def AuthorityCheck(self, username):
table = 'storageUser'
rows, errno, why = ESSMSSQL.DB().action(
table, 'GET3',
('userIdentifierValue', 'userPassword', 'userRight'),
('userIdentifierValue', username))
if not errno:
if len(rows) == 1:
return [1, rows[0][0], rows[0][1], rows[0][2]], 0, str(why)
elif len(rows) > 1:
return [
0, username, 'none', 'none'
], 2, 'Username %s is not unique! dbres: %s , why: %s' % (
username, str(rows), str(why))
else:
return [0, username, 'none', 'none'], 0, str(why)
else:
return [0, 'none', 'none', 'none'], 1, str(why)
def MediumStatusCheck(self, storageMediumID):
table = 'storageMedium'
rows, errno, why = ESSMSSQL.DB().action(
table, 'GET3',
('storageMediumLocationStatus', 'storageMediumLocation'),
('storageMediumID', storageMediumID))
if not errno:
if len(rows) == 1:
return [rows[0][0], rows[0][1], storageMediumID], 0, str(why)
elif len(rows) > 1:
return [
0, storageMediumID
], 2, 'storageMediumID %s is not unique! dbres: %s , why: %s' % (
storageMediumID, str(rows), str(why))
else:
return [
0, storageMediumID
], 3, 'storageMediumID %s not found! dbres: %s , why: %s' % (
storageMediumID, str(rows), str(why))
else:
return [0, 'none'], 1, str(why)
def DeliveryCheck(self, storageMediumID, storagemediumlocation):
table = 'eventStorageMediumSource'
rows, errno, why = ESSMSSQL.DB().action(
table, 'GET3', ('eventIdentifierValue', ),
('eventType', '10', 'AND', 'eventOutcome', '0', 'AND',
'storageMediumID', storageMediumID, 'AND',
'storageMediumDestination', storagemediumlocation))
if not errno:
if len(rows) == 1:
return [rows[0][0], storageMediumID], 0, str(why)
elif len(rows) > 1:
return [
None, storageMediumID
], 2, 'storageMediumID %s is not unique! dbres: %s , why: %s' % (
storageMediumID, str(rows), str(why))
else:
return [
None, storageMediumID
], 3, 'storageMediumID %s not found! dbres: %s , why: %s' % (
storageMediumID, str(rows), str(why))
else:
return [None, 'none'], 1, str(why)
def ThreadMain(self, ProcName):
logging.info('Starting ' + ProcName)
self.tz = timezone.get_default_timezone()
self.ChecksumAlgorithm_CHOICES_dict = dict(ChecksumAlgorithm_CHOICES)
self.ChecksumAlgorithm_CHOICES_invdict = ESSPGM.Check().invert_dict(
self.ChecksumAlgorithm_CHOICES_dict)
while 1:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
('Time', 'Run'),
('Name', ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
self.RunFlag = 0
self.mLock.release()
if Debug: logging.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
('Value', ),
('Name', 'IngestTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
#if Debug: logging.info('Start to list worklist (self.dbget)')
self.dbget, errno, why = ESSDB.DB().action(
self.IngestTable, 'GET4',
('ObjectIdentifierValue', 'ObjectPackageName', 'PolicyId',
'MetaObjectIdentifier', 'MetaObjectSize', 'DataObjectSize',
'ObjectSize', 'ObjectUUID'),
('StatusActivity', '=', '0', 'AND', 'StatusProcess', 'BETWEEN',
49, 'AND', 51))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[0] == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1] == 1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action(
'ESSProc', 'GET', ('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[1] == 1:
logging.info('Process is in pause state')
else:
break
self.ObjectIdentifierValue = self.obj[0]
self.ObjectPackageName = self.obj[1]
self.PolicyId = self.obj[2]
#self.MetaObjectIdentifier = self.obj[3] #METS filename
#self.MetaObjectSize = self.obj[4] #METS size (bytes)
#self.DataObjectSize = self.obj[5]
self.ObjectSize = self.obj[6]
self.ObjectUUID = self.obj[7]
logging.debug('self.obj: ' + str(self.obj))
self.ok = 1
ArchivePolicy_obj = ArchivePolicy.objects.get(
PolicyStat=1, PolicyID=self.PolicyId)
if self.ok:
###########################################################
# set variables
self.AIPpath = ArchivePolicy_obj.AIPpath
self.metatype = ArchivePolicy_obj.IngestMetadata
self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
self.SIPpath = ArchivePolicy_obj.IngestPath
self.ValidateChecksum = ArchivePolicy_obj.ValidateChecksum
self.ValidateXML = ArchivePolicy_obj.ValidateXML
self.ObjectPath = os.path.join(self.AIPpath,
self.ObjectPackageName)
self.Pmets_objpath = os.path.join(
self.AIPpath,
self.ObjectIdentifierValue + '_Package_METS.xml')
if self.metatype > 0:
self.startVerTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 50, 5)
if errno:
logging.error('Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
logging.info('Start validate AIP package: ' +
self.ObjectIdentifierValue)
self.Cmets_obj = None
self.Cmets_objpath = None
self.premis_obj = None
self.premis_objpath = None
self.addml_obj = None
self.addml_objpath = None
##########################################
# Get PMETS info
if self.ok:
[self.Package_info,
self.CMets_info], errno, why = ESSMD.getPMETSInfo(
FILENAME=self.Pmets_objpath)
if errno:
self.event_info = 'Failed to get PMETS info for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug(
'Object: %s, Package_info: %s, CMets_info: %s',
self.ObjectIdentifierValue,
str(self.Package_info), str(self.CMets_info))
# CMets_info and Package_info: ['A0007600_Content_METS.xml', 'MD5', 'b0270cb4d196b72b87fe27ce6242df18', 64058, 'text/xml']
#if self.Package_info[1] == 'MD5': self.PackageMessageDigestAlgorithm = 1
self.PackageMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[
self.Package_info[1]]
if self.Package_info[2]:
self.PackageMessageDigest = self.Package_info[
2]
if self.Package_info[3]:
self.PackageSize = int(self.Package_info[3])
self.Cmets_obj = self.CMets_info[0]
if self.metatype in [1, 2, 3]:
self.Cmets_objpath = os.path.join(
self.AIPpath, self.Cmets_obj)
elif self.metatype in [4]:
self.Cmets_objpath = os.path.join(
self.SIPpath, self.Cmets_obj)
#if self.CMets_info[1] == 'MD5': self.CMetsMessageDigestAlgorithm = 1
self.CMetsMessageDigestAlgorithm = self.ChecksumAlgorithm_CHOICES_invdict[
self.CMets_info[1]]
if self.CMets_info[2]:
self.CMetsMessageDigest = self.CMets_info[2]
if self.CMets_info[3]:
self.CMetsSize = int(self.CMets_info[3])
##########################################################
# Check if ObjectPath and Cmets_objpath exist
if os.path.exists(self.ObjectPath) and os.path.exists(
self.Cmets_objpath):
self.ok = 1
else:
self.event_info = 'The path to Object: %s or METS_Metaobject: %s is not accessible!' % (
self.ObjectPath, self.Cmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
##########################################
# Get MetsFgrp001TotalSize from METSfile
if self.ok:
self.MetsFgrp001TotalSize, errno, why = ESSMD.getFileSizeFgrp001(
FILENAME=self.Cmets_objpath)
if errno:
self.event_info = 'Failed to get MetsFgrp001TotalSize for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug(
'Object: %s, MetsFgrp001TotalSize: %s',
self.ObjectIdentifierValue,
str(self.MetsFgrp001TotalSize))
##########################################
# Get PremisObjectTotalSize from METSfile
if self.ok:
if self.metatype in [1, 2, 3]:
self.premis_objpath = self.Cmets_objpath
elif self.metatype in [4]:
# Get metadata from METS file
res_info, res_files, res_struct, errno, why = ESSMD.getMETSFileList(
FILENAME=self.Cmets_objpath)
if errno:
self.event_info = 'Failed to get metadata from content METS for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
for res_file in res_files:
if res_file[0] == 'amdSec' and res_file[
2] == 'digiprovMD':
self.premis_obj = res_file[8][5:]
self.premis_objpath = '%s/%s/%s' % (
self.SIPpath,
self.ObjectIdentifierValue,
self.premis_obj)
elif res_file[0] == 'amdSec' and res_file[
2] == 'techMD' and res_file[
16] == 'ADDML':
self.addml_obj = res_file[8][5:]
self.addml_objpath = '%s/%s/%s' % (
self.SIPpath,
self.ObjectIdentifierValue,
self.addml_obj)
self.PremisObjectTotalSize, errno, why = ESSMD.getFileSizePremis(
FILENAME=self.premis_objpath)
if errno:
self.event_info = 'Failed to get PREMISobjects total size in METSfile for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug(
'Object: %s, PremisObjectTotalSize: %s',
self.ObjectIdentifierValue,
str(self.PremisObjectTotalSize))
###################################################
# Check if MetsFgrp001TotalSize is equal to PremisObjectTotalSize
if self.ok:
if self.MetsFgrp001TotalSize == self.PremisObjectTotalSize:
logging.info(
'Succeeded to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is equal to PremisObjectTotalSize: %s',
self.ObjectIdentifierValue,
str(self.MetsFgrp001TotalSize),
str(self.PremisObjectTotalSize))
else:
self.event_info = 'Failed to verify METS Fgrp001 and PREMIS for object: %s, MetsFgrp001TotalSize: %s is not equal to PremisObjectTotalSize: %s' % (
self.ObjectIdentifierValue,
str(self.MetsFgrp001TotalSize),
str(self.PremisObjectTotalSize))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
##########################################
# Get CMetsTotalSize
if self.ok:
self.CMetsTotalSize, errno, why = ESSMD.getTotalSize(
FILENAME=self.Cmets_objpath)
if errno:
self.event_info = 'Failed to get CMetsTotalSize for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug('Object: %s, raw CMetsTotalSize: %s',
self.ObjectIdentifierValue,
str(self.CMetsTotalSize))
if self.CMetsSize:
# Add Content Mets filesize
self.CMetsTotalSize[0] += 1
self.CMetsTotalSize[1] += self.CMetsSize
logging.debug('Object: %s, CMetsTotalSize: %s',
self.ObjectIdentifierValue,
str(self.CMetsTotalSize))
##########################################
# Get TarFileSize
if self.ok:
self.TarFileSize, errno, why = ESSPGM.Check(
).getFileSizeTAR(self.ObjectPath)
if errno:
self.event_info = 'Failed to get TarFileSize for object: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug('Object: %s, TarFileSize: %s',
self.ObjectIdentifierValue,
str(self.TarFileSize))
###################################################
# Check if CMetsTotalSize is equal to TarFileSize
if self.ok:
if self.CMetsTotalSize == self.TarFileSize:
logging.info(
'Succeeded to verify total package size for object: %s, CMetsTotalSize: %s is equal to TarFileSize: %s',
self.ObjectIdentifierValue,
str(self.CMetsTotalSize),
str(self.TarFileSize))
else:
self.event_info = 'Failed to verify total package size for object: %s, CMetsTotalSize: %s is not equal to TarFileSize: %s' % (
self.ObjectIdentifierValue,
str(self.CMetsTotalSize), str(
self.TarFileSize))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
########################
# get checksum for PMETS file
if self.ok:
self.PMetsMessageDigestAlgorithm = self.ChecksumAlgorithm
self.PMetsMessageDigest = ''
self.Pmets_obj_checksum, errno, why = ESSPGM.Check(
).checksum(
self.Pmets_objpath,
self.PMetsMessageDigestAlgorithm) # Checksum
if errno:
self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
self.Cmets_objpath, str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug('Checksum for PMETS file: %s',
self.Pmets_obj_checksum)
self.PMetsMessageDigestAlgorithm = 1
self.PMetsMessageDigest = self.Pmets_obj_checksum
########################
# get checksum for CMETS file
if self.ok:
self.Cmets_obj_checksum, errno, why = ESSPGM.Check(
).checksum(
self.Cmets_objpath,
self.CMetsMessageDigestAlgorithm) # Checksum
if errno:
self.event_info = 'Failed to get checksum for: %s, Error: %s' % (
self.Cmets_objpath, str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.debug('Checksum for CMETS file: %s',
self.Cmets_obj_checksum)
###################################################
# Check CMetsMessageDigest
if self.ok:
if self.ValidateChecksum:
if self.CMetsMessageDigest == self.Cmets_obj_checksum:
logging.info(
'Succeeded to verify Content Mets MessageDigest for object: %s',
self.ObjectIdentifierValue)
else:
self.event_info = 'Failed to verify Content Mets MessageDigest for object: %s, CMetsMessageDigest: %s is equal to FileMessageDigest: %s' % (
self.ObjectIdentifierValue,
str(self.CMetsMessageDigest),
str(self.Cmets_obj_checksum))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.warning(
'Checksum validate is disabled for object: %s',
self.ObjectIdentifierValue)
if self.ValidateXML:
########################
# XML Schema Validate CMETS file
if self.ok:
errno, why = ESSMD.validate(
FILENAME=self.Cmets_objpath)
if errno:
self.event_info = 'Failed to schema validate Content METS file for object: %s, why: %s' % (
self.ObjectIdentifierValue, str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to schema validate Content METS file for object: %s',
self.ObjectIdentifierValue)
########################
# XML Schema Validate PMETS file
if self.ok:
errno, why = ESSMD.validate(
FILENAME=self.Pmets_objpath)
if errno:
self.event_info = 'Failed to schema validate Package METS file for object: %s, why: %s' % (
self.ObjectIdentifierValue, str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to schema validate Package METS file for object: %s',
self.ObjectIdentifierValue)
########################
# XML Schema Validate PREMIS file
if self.ok and self.premis_obj:
errno, why = ESSMD.validate(
FILENAME=self.premis_objpath)
if errno:
self.event_info = 'Failed to schema validate PREMIS file for object: %s, why: %s' % (
self.ObjectIdentifierValue, str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to schema validate PREMIS file for object: %s',
self.ObjectIdentifierValue)
########################
# XML Schema Validate ADDML file
if self.ok and self.addml_obj:
errno, why = ESSMD.validate(
FILENAME=self.addml_objpath)
if errno:
self.event_info = 'Failed to schema validate ADDML file for object: %s, why: %s' % (
self.ObjectIdentifierValue, str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to schema validate ADDML file for object: %s',
self.ObjectIdentifierValue)
else:
logging.warning(
'Schema validate XML is disabled for object: %s',
self.ObjectIdentifierValue)
self.stopVerTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
self.VerTime = self.stopVerTime - self.startVerTime
if self.VerTime.seconds < 1:
self.VerTime = datetime.timedelta(
seconds=1
) #Fix min time to 1 second if it is zero.
self.ObjectSizeMB = int(self.ObjectSize) / 1048576
self.VerMBperSEC = int(self.ObjectSizeMB) / int(
self.VerTime.seconds)
##################################
# Write metadatafiles to DB-blob or FTP
if self.ok:
if ESSDB.DB().action('ESSConfig', 'GET', ('Value', ),
('Name', 'MD_FTP_HOST'))[0][0]:
if self.Cmets_obj:
##################################
# Write CMETS metadatafile to DB-blob
res, errno, why = ESSmetablob.prod(
).StoreMetadataBlob(
ObjectUUID=self.ObjectUUID,
ObjectIdentifierValue=self.
ObjectIdentifierValue,
ObjectMetadataType=26,
FILENAME=self.Cmets_objpath,
FTPFileName=string.replace(
self.Cmets_obj,
self.ObjectIdentifierValue + '/', ''),
FTPflag=1,
DBflag=0,
AgentIdentifierValue=AgentIdentifierValue)
if errno:
self.event_info = 'Failed to store Content METS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to store Content METS file to FTP server or DB-blob for object: %s',
self.ObjectIdentifierValue)
if self.premis_obj:
##################################
# Write PREMIS metadatafile to DB-blob
res, errno, why = ESSmetablob.prod(
).StoreMetadataBlob(
ObjectUUID=self.ObjectUUID,
ObjectIdentifierValue=self.
ObjectIdentifierValue,
ObjectMetadataType=27,
FILENAME=self.premis_objpath,
FTPFileName=self.premis_obj,
FTPflag=1,
DBflag=0,
AgentIdentifierValue=AgentIdentifierValue)
if errno:
self.event_info = 'Failed to store PREMIS file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to store PREMIS file to FTP server or DB-blob for object: %s',
self.ObjectIdentifierValue)
if self.addml_obj:
##################################
# Write ADDML metadatafile to DB-blob
res, errno, why = ESSmetablob.prod(
).StoreMetadataBlob(
ObjectUUID=self.ObjectUUID,
ObjectIdentifierValue=self.
ObjectIdentifierValue,
ObjectMetadataType=25,
FILENAME=self.addml_objpath,
FTPFileName=self.addml_obj,
FTPflag=1,
DBflag=0,
AgentIdentifierValue=AgentIdentifierValue)
if errno:
self.event_info = 'Failed to store ADDML file to FTP server or DB-blob: %s, errno: %s, why: %s' % (
self.ObjectIdentifierValue, str(errno),
str(why))
logging.error(self.event_info)
ESSPGM.Events().create(
'1050', '', 'ESSArch AIPValidate',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
else:
logging.info(
'Succeeded to store ADDML file to FTP server or DB-blob for object: %s',
self.ObjectIdentifierValue)
else:
logging.info(
'Skip to store metadata to FTP server or DB-blob for object: %s',
self.ObjectIdentifierValue)
if not self.ok:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 51, 4)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Failed to validate AIP package: ' + self.ObjectIdentifierValue
logging.error(self.event_info)
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
else:
logging.info('Succeeded to validate AIP package: ' +
self.ObjectIdentifierValue + ' , ' +
str(self.VerMBperSEC) +
' MB/Sec and Time: ' + str(self.VerTime))
self.timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(
self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('StatusProcess', '59', 'StatusActivity', '0',
'CMetaMessageDigestAlgorithm',
self.CMetsMessageDigestAlgorithm,
'CMetaMessageDigest', self.CMetsMessageDigest,
'PMetaMessageDigestAlgorithm',
self.PMetsMessageDigestAlgorithm,
'PMetaMessageDigest', self.PMetsMessageDigest,
'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue, 'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
else:
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '0', '', 2,
self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('StatusProcess', '59', 'StatusActivity', '0',
'CMetaMessageDigestAlgorithm',
self.CMetsMessageDigestAlgorithm,
'CMetaMessageDigest', self.CMetsMessageDigest,
'PMetaMessageDigestAlgorithm',
self.PMetsMessageDigestAlgorithm,
'PMetaMessageDigest', self.PMetsMessageDigest,
'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error(
'Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error(
'Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
elif self.metatype == 0: #self.metatype 0 = No metadata
logging.info('Skip to validate AIP package: ' +
self.ObjectIdentifierValue)
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('StatusProcess', '59', 'StatusActivity', '0',
'CMetaMessageDigestAlgorithm', '0',
'PMetaMessageDigestAlgorithm', '0', 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue,
'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
else:
ESSPGM.Events().create('1050', '',
'ESSArch AIPValidate',
ProcVersion, '0',
'Skip to validate AIP package',
2, self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('StatusProcess', '59', 'StatusActivity', '0',
'CMetaMessageDigestAlgorithm', '0',
'PMetaMessageDigestAlgorithm', '0',
'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag = 0
def form_valid(self, form):
timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,
tzinfo=pytz.utc)
timestamp_dst = timestamp_utc.astimezone(
timezone.get_default_timezone())
logger = logging.getLogger('essarch.storagemaintenance')
ReqPurpose = form.cleaned_data['ReqPurpose']
if self.request.is_ajax():
MediumList = []
for o in json.loads(form.cleaned_data['MediumList']):
MediumList.append(o[0])
else:
MediumList = form.cleaned_data['MediumList'].split(' ')
#print 'MediumList: %s' % MediumList
storageMedium_objs = storageMedium.objects.filter(
storageMediumID__in=MediumList)
#print len(storageMedium_objs)
for storageMedium_obj in storageMedium_objs:
storageMedium_obj.storageMediumStatus = 0
storageMedium_obj.LocalDBdatetime = timestamp_utc
storageMedium_obj.save(
update_fields=['storageMediumStatus', 'LocalDBdatetime'])
event_info = 'Setting mediumstatus to inactive for media: %s, ReqPurpose: %s' % (
storageMedium_obj.storageMediumID, ReqPurpose)
logger.info(event_info)
ESSPGM.Events().create(
'2090',
'',
'Storage maintenance',
__version__,
'0',
event_info,
2,
storageMediumID=storageMedium_obj.storageMediumID)
if ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
'storageMedium', 'UPD',
('storageMediumStatus',
storageMedium_obj.storageMediumStatus),
('storageMediumID', storageMedium_obj.storageMediumID))
if ext_errno:
logger.error('Failed to update External DB: ' +
str(storageMedium_obj.storageMediumID) +
' error: ' + str(ext_why))
else:
storageMedium_obj.ExtDBdatetime = timestamp_utc
storageMedium_obj.save(update_fields=['ExtDBdatetime'])
robotMediumList = robot.objects.filter(t_id__in=MediumList)
for media in robotMediumList:
media.status = 'Inactive'
media.save(update_fields=['status'])
event_info_robot = 'Setting status to Inactive for media: %s, ReqPurpose: %s' % (
media.t_id, ReqPurpose)
logger.info(event_info_robot)
if self.request.is_ajax():
'''Render Datatables expected JSON format'''
data = {
'sEcho': 'OK',
}
return self.json_response(data)
else:
return super(DeactivateMedia, self).form_valid(form)
def update_archive_obj_from_ais(ais=True):
ip_objs = ArchiveObject.objects.filter(
Q(DELIVERYTYPE__isnull=True) | Q(ObjectUUID__isnull=True)
| Q(ObjectPackageName__isnull=True))
for ip_obj in ip_objs:
if ais:
ip_objs_ext, errno3, why3 = ESSMSSQL.DB().action(
'IngestObject', 'GET3',
('PolicyId', 'ObjectIdentifierValue', 'ObjectPackageName',
'ObjectSize', 'ObjectNumItems',
'ObjectMessageDigestAlgorithm', 'ObjectMessageDigest',
'ObjectPath', 'MetaObjectIdentifier', 'MetaObjectSize',
'CMetaMessageDigestAlgorithm', 'CMetaMessageDigest',
'PMetaMessageDigestAlgorithm', 'PMetaMessageDigest',
'DataObjectSize', 'DataObjectNumItems', 'Status',
'StatusActivity', 'StatusProcess', 'LastEventDate',
'linkingAgentIdentifierValue', 'CreateDate',
'CreateAgentIdentifierValue', 'objectGuid', 'EntryDate',
'EntryAgentIdentifierValue', 'OAISPackageType',
'preservationLevelValue', 'DELIVERYTYPE', 'INFORMATIONCLASS',
'ObjectActive'),
('ObjectIdentifierValue', ip_obj.ObjectIdentifierValue))
else:
# start - test without centralDB
ip_objs_ext = []
if not ip_obj.ObjectUUID: ip_obj.ObjectUUID = str(uuid.uuid4())
if not ip_obj.ObjectIdentifierValue:
ip_obj.ObjectIdentifierValue = ''
if not ip_obj.ObjectPackageName: ip_obj.ObjectPackageName = ''
if not ip_obj.ObjectMessageDigest: ip_obj.ObjectMessageDigest = ''
if not ip_obj.ObjectPath: ip_obj.ObjectPath = ''
if not ip_obj.MetaObjectIdentifier:
ip_obj.MetaObjectIdentifier = ''
if not ip_obj.CMetaMessageDigest: ip_obj.CMetaMessageDigest = ''
if not ip_obj.PMetaMessageDigest: ip_obj.PMetaMessageDigest = ''
if not ip_obj.linkingAgentIdentifierValue:
ip_obj.linkingAgentIdentifierValue = ''
if not ip_obj.CreateAgentIdentifierValue:
ip_obj.CreateAgentIdentifierValue = ''
if not ip_obj.EntryAgentIdentifierValue:
ip_obj.EntryAgentIdentifierValue = ''
if not ip_obj.DELIVERYTYPE: ip_obj.DELIVERYTYPE = ''
if not ip_obj.OAISPackageType: ip_obj.OAISPackageType = 2
if not ip_obj.EntryDate:
ip_obj.EntryDate = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
ip_obj_ext = [ #ip_obj.PolicyId.PolicyID,
'99', ip_obj.ObjectIdentifierValue, ip_obj.ObjectPackageName,
ip_obj.ObjectSize, ip_obj.ObjectNumItems,
ip_obj.ObjectMessageDigestAlgorithm,
ip_obj.ObjectMessageDigest, ip_obj.ObjectPath,
ip_obj.MetaObjectIdentifier, ip_obj.MetaObjectSize,
ip_obj.CMetaMessageDigestAlgorithm, ip_obj.CMetaMessageDigest,
ip_obj.PMetaMessageDigestAlgorithm, ip_obj.PMetaMessageDigest,
ip_obj.DataObjectSize, ip_obj.DataObjectNumItems,
ip_obj.Status, ip_obj.StatusActivity, ip_obj.StatusProcess,
ip_obj.LastEventDate, ip_obj.linkingAgentIdentifierValue,
ip_obj.CreateDate, ip_obj.CreateAgentIdentifierValue,
ip_obj.ObjectUUID, ip_obj.EntryDate,
ip_obj.EntryAgentIdentifierValue, ip_obj.OAISPackageType,
ip_obj.preservationLevelValue, ip_obj.DELIVERYTYPE,
ip_obj.INFORMATIONCLASS, ip_obj.ObjectActive
]
ip_objs_ext.append(ip_obj_ext)
errno3 = 0
why3 = ''
# end - test without centralDB
if errno3:
logger.error('Failed to access central DB ' + str(why3))
continue
elif len(ip_objs_ext) == 1:
ip_obj_ext = ip_objs_ext[0]
logger.debug('IngestObjectUUID: %s' %
str(uuid.UUID(ip_obj_ext[23])))
if ip_obj.check_db_sync(
): # Check if ESSArch and centralDB is in sync
###################################################
# archive object exist in local "IngestObject" DB try to update
#logger.info('Found archive object: %s in local "IngestObject" DB, try to update',ip_obj_ext[1])
timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
timestamp_dst = timestamp_utc.astimezone(tz)
#ArchivePolicy_obj = ArchivePolicy.objects.get(PolicyID = ip_obj_ext[0])
#ip_obj.PolicyId = ArchivePolicy_obj
update_values = {}
update_values['ais_values_set_to_blank'] = []
update_values['ais_values_to_check'] = []
update_values['local_values_set_to_blank'] = []
update_values['local_values_updated'] = []
update_local_ip = True
ip_obj, update_values = update_or_log(ip_obj,
'ObjectPackageName',
ip_obj_ext[2],
update_values)
ip_obj.ObjectSize = ip_obj_ext[3]
ip_obj.ObjectNumItems = ip_obj_ext[4]
ip_obj.ObjectMessageDigestAlgorithm = ip_obj_ext[5]
ip_obj, update_values = update_or_log(ip_obj,
'ObjectMessageDigest',
ip_obj_ext[6],
update_values)
ip_obj, update_values = update_or_log(ip_obj, 'ObjectPath',
ip_obj_ext[7],
update_values)
ip_obj, update_values = update_or_log(ip_obj,
'MetaObjectIdentifier',
ip_obj_ext[8],
update_values)
ip_obj.MetaObjectSize = ip_obj_ext[9]
ip_obj.CMetaMessageDigestAlgorithm = ip_obj_ext[10]
ip_obj, update_values = update_or_log(ip_obj,
'CMetaMessageDigest',
ip_obj_ext[11],
update_values)
ip_obj.PMetaMessageDigestAlgorithm = ip_obj_ext[12]
ip_obj, update_values = update_or_log(ip_obj,
'PMetaMessageDigest',
ip_obj_ext[13],
update_values)
ip_obj.DataObjectSize = ip_obj_ext[14]
ip_obj.DataObjectNumItems = ip_obj_ext[15]
ip_obj.Status = ip_obj_ext[16]
ip_obj.StatusActivity = ip_obj_ext[17]
ip_obj.StatusProcess = ip_obj_ext[18]
ip_obj, update_values = datetime_update_or_log(
ip_obj, 'LastEventDate', ip_obj_ext[19], update_values)
ip_obj, update_values = update_or_log(
ip_obj, 'linkingAgentIdentifierValue', ip_obj_ext[20],
update_values)
ip_obj, update_values = datetime_update_or_log(
ip_obj, 'CreateDate', ip_obj_ext[21], update_values)
ip_obj, update_values = update_or_log(
ip_obj, 'CreateAgentIdentifierValue', ip_obj_ext[22],
update_values)
ip_obj.ObjectUUID = uuid.UUID(ip_obj_ext[23])
ip_obj, update_values = datetime_update_or_log(
ip_obj, 'EntryDate', ip_obj_ext[24], update_values)
ip_obj, update_values = update_or_log(
ip_obj, 'EntryAgentIdentifierValue', ip_obj_ext[25],
update_values)
ip_obj.OAISPackageType = ip_obj_ext[26]
ip_obj.preservationLevelValue = ip_obj_ext[27]
ip_obj, update_values = update_or_log(ip_obj, 'DELIVERYTYPE',
ip_obj_ext[28],
update_values)
ip_obj.INFORMATIONCLASS = ip_obj_ext[29]
ip_obj.ObjectActive = ip_obj_ext[30]
ip_obj.LocalDBdatetime = timestamp_utc
ip_obj.ExtDBdatetime = timestamp_utc
if update_values['ais_values_set_to_blank']:
logger.warning(
'Need to manual set fields: %s to blank is AIS for object: %s'
% (repr(update_values['ais_values_set_to_blank']),
ip_obj.ObjectIdentifierValue))
if update_values['ais_values_to_check']:
update_local_ip = False
logger.error(
'Need to manual check values for fields: %s in AIS for object: %s'
% (repr(update_values['ais_values_to_check']),
ip_obj.ObjectIdentifierValue))
if update_values['local_values_set_to_blank']:
logger.warning(
'local fields: %s updated to blank for object: %s' %
(repr(update_values['local_values_set_to_blank']),
ip_obj.ObjectIdentifierValue))
if update_values['local_values_updated']:
logger.warning(
'local fields: %s updated to new values for object: %s'
% (repr(update_values['local_values_updated']),
ip_obj.ObjectIdentifierValue))
if update_local_ip:
logger.info(
'Found archive object: %s in local "IngestObject" DB, try to update',
ip_obj_ext[1])
ip_obj.save()
else:
logger.error(
'Skip to update local object: %s, local DB and central DB is not in sync'
% ip_obj.ObjectIdentifierValue)
else:
logger.error(
'Local DB and central DB is not in sync for object: %s' %
ip_obj.ObjectIdentifierValue)
continue
else:
logger.error(
'Problem to get information about object: %s from central DB' %
ip_obj.ObjectIdentifierValue)
continue
def ObjectValidate(self):
self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET', ('Value', ),
('Name', 'IngestTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
# Check if exist extDB and got projektid
self.dbget, errno, why = ESSDB.DB().action(
self.IngestTable, 'GET4',
('ObjectIdentifierValue', 'PolicyID', 'StatusProcess',
'StatusActivity', 'ObjectUUID'),
('StatusActivity', '=', '0', 'AND', 'StatusProcess', 'BETWEEN', 9,
'AND', 14, 'OR', 'StatusActivity', '=', '4', 'AND',
'StatusProcess', 'BETWEEN', 10, 'AND', 11))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ObjectIdentifierValue = self.obj[0]
self.PolicyID = self.obj[1]
self.StatusProcess = self.obj[2]
self.StatusActivity = self.obj[3]
ObjectUUID = self.obj[4]
self.DBmode = 0
self.ext_ObjectGuid = ObjectUUID
self.ext_EntryDate = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.ext_EntryAgentIdentifierValue = None
self.ext_OAISPackageType = 2
self.ext_preservationLevelValue = 1
self.ext_ObjectActive = 0
self.objectstatus = 0
self.ext_ProjectGroupCode = ''
self.ext_ObjectPackageName = ''
if Debug:
logging.info('StatusProcess 9, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
#Check....
if self.PolicyID:
ArchivePolicy_objs = ArchivePolicy.objects.filter(
PolicyStat=1, PolicyID=str(self.PolicyID))[:1]
if ArchivePolicy_objs:
ArchivePolicy_obj = ArchivePolicy_objs.get()
if ArchivePolicy_obj.Mode in range(0, 2):
self.DBmode = ArchivePolicy_obj.Mode
logging.info(
'Policy found for Object: %s in ESSArch mode' %
self.ObjectIdentifierValue)
if ArchivePolicy_obj.IngestMetadata in [1, 2, 3]:
metsfilename = os.path.join(
ArchivePolicy_obj.IngestPath,
self.ObjectIdentifierValue +
'_Package_METS.xml')
elif ArchivePolicy_obj.IngestMetadata in [4]:
ObjectPath = os.path.join(
ArchivePolicy_obj.IngestPath,
self.ObjectIdentifierValue)
if os.path.exists(
os.path.join(ObjectPath, 'sip.xml')):
metsfilename = os.path.join(
ObjectPath, 'sip.xml')
elif os.path.exists(
os.path.join(ObjectPath, 'mets.xml')):
metsfilename = os.path.join(
ObjectPath, 'mets.xml')
#elif os.path.exists(os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
# metsfilename = os.path.join(ObjectPath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)
else:
metsfilename = ''
#metsfilename = '%s/sip.xml' % os.path.join(ArchivePolicy_obj.IngestPath,self.ObjectIdentifierValue)
else:
metsfilename = ''
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=metsfilename)
if not error:
# cut off microsecond and timezone info ".xxxxxxx+02:00"
#for c in res_info[1][0]:
# if c == '.' or c == '+':
# break
# else:
# self.ext_EntryDate += c
self.ext_EntryDate = parse_datetime(
res_info[1][0]).astimezone(pytz.utc)
self.ext_EntryAgentIdentifierValue = res_info[2][
0][4]
try:
self.ext_ObjectGuid = str(
uuid.UUID(self.ObjectIdentifierValue))
except ValueError, why:
logging.warning(
'ObjectIdentifierValue: %s is not a valid UUID, why: %s , start to generate a new UUID'
% (self.ObjectIdentifierValue, str(why)))
self.ext_ObjectGuid = str(uuid.uuid1())
logging.info(
'New UUID: %s for ObjectIdentifierValue: %s'
% (self.ext_ObjectGuid,
str(self.ObjectIdentifierValue)))
self.ext_ObjectActive = 1
self.ext_OAISPackageType = 2
self.ext_preservationLevelValue = 1
self.objectstatus = 1
else:
self.objectstatus = 102 # Problem to get information from package METS
elif ArchivePolicy_obj.Mode == 2: # AIS but POLICYID from METS, Check in AIS if object is active.
self.DBmode = ArchivePolicy_obj.Mode
self.extOBJdbget, ext_errno, ext_why = ESSMSSQL.DB(
).action(self.IngestTable, 'GET3',
('ProjectGroupCode', 'ObjectPackageName',
'ObjectGuid', 'ObjectActive', 'EntryDate',
'EntryAgentIdentifierValue',
'OAISPackageType', 'preservationLevelValue'),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
#self.extOBJdbget = [[10,'','7283074a-00c0-11e2-a78f-002215836500',1,'2010-07-12 16:57:45','entryagent',2,1]]
#ext_errno = 0
#ext_why = 'whywhy'
if ext_errno:
logging.error('Failed to access External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
elif self.extOBJdbget:
if Debug:
logging.info(
'Found object: %s in AIS, self.extOBJdbget: %s',
self.ObjectIdentifierValue,
str(self.extOBJdbget))
self.objectstatus = 10 # Object found in external DB
if self.objectstatus < 100:
########################################
# Check if object alredy have an AIP
if disable_ObjectPackageName:
self.ext_ObjectPackageName = ''
else:
self.ext_ObjectPackageName = self.extOBJdbget[
0][1]
if not self.ext_ObjectPackageName:
self.objectstatus = 12 # Object do not have an AIP
else:
self.objectstatus = 112 # Object already have an AIP
if self.objectstatus < 100:
########################################
# Get GUID/UUID
#self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2]) #When pymssql
self.ext_ObjectGuid = uuid.UUID(
self.extOBJdbget[0][2])
if self.objectstatus < 100:
########################################
# Check if object is active
self.ext_ObjectActive = self.extOBJdbget[0][3]
if self.ext_ObjectActive == 1:
self.objectstatus = 13 # Object is active
else:
self.objectstatus = 113 # Object is not active
if self.objectstatus < 100:
########################################
# Check if POLICYID in local DB "METS" is equal to ProjectGroupCode in AIS
self.ext_ProjectGroupCode = str(
self.extOBJdbget[0][0])
if self.ext_ProjectGroupCode == self.PolicyID:
self.objectstatus = 1 # Object have an ProjectCode
logging.info(
'Object: %s found in AIS with correct POLICYID'
% self.ObjectIdentifierValue)
else:
self.objectstatus = 111 # Object do not have an ProjectCode
########################################
self.ext_EntryDate = self.extOBJdbget[0][
4].replace(microsecond=0,
tzinfo=self.tz).astimezone(pytz.utc)
self.ext_EntryAgentIdentifierValue = self.extOBJdbget[
0][5]
self.ext_OAISPackageType = self.extOBJdbget[0][6]
self.ext_preservationLevelValue = self.extOBJdbget[
0][7]
########################################
# Special function only for test
if force_ProjectGroupCode:
logging.info(
'Force set ProjectGroupCode for Object: %s'
% self.ObjectIdentifierValue)
self.objectstatus = 10
self.ext_ProjectGroupCode = force_ProjectGroupCode
self.ext_ObjectPackageName = ''
self.ext_ObjectGuid = str(
uuid.uuid1()) # updDB
self.ext_ObjectActive = 1 # updDB
self.ext_EntryDate = datetime.datetime.utcnow(
).replace(microsecond=0,
tzinfo=pytz.utc) # updDB
self.ext_EntryAgentIdentifierValue = None # updDB
self.ext_OAISPackageType = 2 # updDB
self.ext_preservationLevelValue = 1 # updDB
else:
self.objectstatus = 110 # Object not found in external DB
if Debug:
logging.info(
'Missing object: %s in AIS, self.extOBJdbget: %s',
self.ObjectIdentifierValue,
str(self.extOBJdbget))
else:
self.objectstatus = 100 # Policy is not in ESSArch mode
else:
self.objectstatus = 101 # Policy not found or not active
#model.meta.Session.close()
else:
self.DBmode = 2 # AIS
self.extOBJdbget, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'GET3',
('ProjectGroupCode', 'ObjectPackageName', 'ObjectGuid',
'ObjectActive', 'EntryDate', 'EntryAgentIdentifierValue',
'OAISPackageType', 'preservationLevelValue'),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to access External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
elif self.extOBJdbget:
#if not ext_errno and self.extOBJdbget:
if Debug:
logging.info(
'Found object: %s in AIS, self.extOBJdbget: %s',
self.ObjectIdentifierValue, str(self.extOBJdbget))
self.objectstatus = 10 # Object found in external DB
########################################
# Check if object alredy have an AIP
self.ext_ProjectGroupCode = str(self.extOBJdbget[0][0])
if self.objectstatus < 100 and self.ext_ProjectGroupCode:
self.objectstatus = 11 # Object have an ProjectCode
else:
self.objectstatus = 111 # Object do not have an ProjectCode
########################################
# Check if object alredy have an AIP
if disable_ObjectPackageName:
self.ext_ObjectPackageName = ''
else:
self.ext_ObjectPackageName = self.extOBJdbget[0][1]
if self.objectstatus < 100 and not self.ext_ObjectPackageName:
self.objectstatus = 12 # Object do not have an AIP
else:
self.objectstatus = 112 # Object already have an AIP
########################################
# Get GUID/UUID
#self.ext_ObjectGuid = uuid.UUID(bytes_le=self.extOBJdbget[0][2]) #When pymssql
self.ext_ObjectGuid = uuid.UUID(self.extOBJdbget[0][2])
########################################
# Check if object is active
self.ext_ObjectActive = self.extOBJdbget[0][3]
if self.objectstatus < 100 and self.ext_ObjectActive == 1:
self.objectstatus = 13 # Object is active
else:
self.objectstatus = 113 # Object is not active
########################################
self.ext_EntryDate = self.extOBJdbget[0][4].replace(
microsecond=0, tzinfo=self.tz).astimezone(pytz.utc)
self.ext_EntryAgentIdentifierValue = self.extOBJdbget[0][5]
self.ext_OAISPackageType = self.extOBJdbget[0][6]
self.ext_preservationLevelValue = self.extOBJdbget[0][7]
if Debug:
logging.info('ext_ProjectGroupCode is: ' +
str(self.ext_ProjectGroupCode) +
' for ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
if force_ProjectGroupCode:
logging.info(
'Force set ProjectGroupCode for Object: %s' %
self.ObjectIdentifierValue)
self.objectstatus = 10
self.ext_ProjectGroupCode = force_ProjectGroupCode
self.ext_ObjectPackageName = ''
self.ext_ObjectGuid = str(uuid.uuid1()) # updDB
self.ext_ObjectActive = 1 # updDB
self.ext_EntryDate = datetime.datetime.utcnow(
).replace(microsecond=0, tzinfo=pytz.utc) # updDB
self.ext_EntryAgentIdentifierValue = None # updDB
self.ext_OAISPackageType = 2 # updDB
self.ext_preservationLevelValue = 1 # updDB
if self.objectstatus < 100:
ArchivePolicy_objs = ArchivePolicy.objects.filter(
PolicyStat=1,
AISProjectID=self.ext_ProjectGroupCode)[:1]
if ArchivePolicy_objs:
self.PolicyID = ArchivePolicy_objs.get().PolicyID
if Debug:
logging.info('PolicyID: ' + str(self.PolicyID))
self.objectstatus = 14 # Object got PolicyID
else:
self.objectstatus = 114 # Object mising PolicyID
else:
self.objectstatus = 110 # Object not found in external DB
if Debug:
logging.info(
'Missing object: %s in AIS, self.extOBJdbget: %s',
self.ObjectIdentifierValue, str(self.extOBJdbget))
if self.objectstatus == 100: # Policy is not in ESSArch mode
###################################################################
# Policy is not in ESSArch mode(12) and Need of assistance(4)
###################################################################
self.StatusProcess = 12
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 12, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Policy is not in ESSArch mode for Object: ' + str(
self.ObjectIdentifierValue)
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 101: # Policy not found or not active
###################################################################
# Policy not found or active(12) and Need of assistance(4)
###################################################################
self.StatusProcess = 12
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 12, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Policy not found or active for Object: ' + str(
self.ObjectIdentifierValue)
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 102: # Problem to get information from Package_METS
###################################################################
# Problem to get information from Package_METS(12) and Need of assistance(4)
###################################################################
self.StatusProcess = 12
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 12, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Problem to get information from Package_METS for Object: ' + str(
self.ObjectIdentifierValue)
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 110:
###################################################################
#Object don't exist in extDB(10) and Need of assistance(4)
###################################################################
self.StatusProcess = 10
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 10, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
elif self.objectstatus == 111:
###################################################################
#Object don't have any projektkod in extDB(11) and Need of assistance(4)
###################################################################
self.StatusProcess = 11
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 11, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
elif self.objectstatus == 112:
###################################################################
#Object already have an AIP!!
###################################################################
self.StatusProcess = 13
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 13, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Object: ' + str(
self.ObjectIdentifierValue) + ' already have an AIP!'
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 113:
###################################################################
# Object is not active!!
###################################################################
self.StatusProcess = 14
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 14, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Object: ' + str(
self.ObjectIdentifierValue
) + ' is not active in external DB!'
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 114:
###################################################################
#Object don't have any local policy(12) and Need of assistance(4)
###################################################################
self.StatusProcess = 12
self.StatusActivity = 4
if Debug:
logging.info(
'Change to StatusProcess 12, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
self.event_info = 'Object: ' + str(
self.ObjectIdentifierValue
) + ' do not have any local policy!'
logging.error(self.event_info)
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '1', self.event_info,
self.DBmode, self.ObjectIdentifierValue)
elif self.objectstatus == 14 or self.objectstatus == 1:
###################################################################
#Object got a policy(19) and RFNext and OK(0)
###################################################################
self.StatusProcess = 19
self.StatusActivity = 0
if Debug:
logging.info(
'Change to StatusProcess 19, ObjectIdentifierValue ' +
str(self.ObjectIdentifierValue))
ESSPGM.Events().create('1010', '', 'ESSArch SIPValidateAIS',
ProcVersion, '0', '', self.DBmode,
self.ObjectIdentifierValue)
logging.info(
'objectstatus:%s,StatusProcess:%s,StatusActivity:%s,EntryDate:%s'
% (self.objectstatus, self.StatusProcess, self.StatusActivity,
self.ext_EntryDate))
if self.objectstatus:
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('PolicyId', self.PolicyID, 'ObjectUUID',
self.ext_ObjectGuid, 'EntryDate',
self.ext_EntryDate.replace(tzinfo=None),
'EntryAgentIdentifierValue',
self.ext_EntryAgentIdentifierValue, 'StatusProcess',
self.StatusProcess, 'StatusActivity', self.StatusActivity,
'LastEventDate', self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue,
'OAISPackageType', self.ext_OAISPackageType,
'preservationLevelValue', self.ext_preservationLevelValue,
'ObjectActive', self.ext_ObjectActive, 'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if errno == 0 and self.ext_IngestTable:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('PolicyId', self.PolicyID, 'StatusProcess',
self.StatusProcess, 'StatusActivity',
self.StatusActivity, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
def ThreadMain(self, ProcName):
logging.info('Starting ' + ProcName)
TimeZone = timezone.get_default_timezone_name()
self.tz = pytz.timezone(TimeZone)
METS_NAMESPACE = SchemaProfile.objects.get(
entity='mets_namespace').value
METS_SCHEMALOCATION = SchemaProfile.objects.get(
entity='mets_schemalocation').value
METS_PROFILE = SchemaProfile.objects.get(entity='mets_profile').value
XLINK_NAMESPACE = SchemaProfile.objects.get(
entity='xlink_namespace').value
XSI_NAMESPACE = SchemaProfile.objects.get(entity='xsi_namespace').value
while 1:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
('Time', 'Run'),
('Name', ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
self.RunFlag = 0
self.mLock.release()
if Debug: logging.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
Cmets_obj = Parameter.objects.get(
entity='content_descriptionfile').value
self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
('Value', ),
('Name', 'IngestTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
self.dbget, errno, why = ESSDB.DB().action(
self.IngestTable, 'GET4',
('ObjectIdentifierValue', 'ObjectUUID', 'PolicyId',
'ObjectSize'), ('StatusProcess', 'BETWEEN', 39, 'AND', 40,
'AND', 'StatusActivity', '=', '0'))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ok = 1
self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[0] == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1] == 1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action(
'ESSProc', 'GET', ('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[1] == 1:
logging.info('Process is in pause state')
else:
break
self.ObjectIdentifierValue = self.obj[0]
self.ObjectUUID = self.obj[1]
self.PolicyId = self.obj[2]
self.ObjectSize = self.obj[3]
ArchivePolicy_obj = ArchivePolicy.objects.get(
PolicyStat=1, PolicyID=self.PolicyId)
if self.ok:
###########################################################
# set variables
self.AIPpath = ArchivePolicy_obj.AIPpath
self.metatype = ArchivePolicy_obj.IngestMetadata
self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
self.CA = dict(ChecksumAlgorithm_CHOICES)[
self.ChecksumAlgorithm]
self.SIPpath = ArchivePolicy_obj.IngestPath
self.p_obj = self.ObjectIdentifierValue + '.tar'
self.ObjectPath = os.path.join(self.AIPpath, self.p_obj)
self.SIProotpath = os.path.join(self.SIPpath,
self.ObjectIdentifierValue)
if self.metatype in [4]:
#self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
#self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
#self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
self.Cmets_obj = Cmets_obj.replace(
'{objid}', self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.SIProotpath, self.Cmets_obj)
elif self.metatype in [1, 2, 3]:
self.Cmets_obj = '%s_Content_METS.xml' % (
self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.AIPpath, self.Cmets_obj)
self.Pmets_obj = '%s_Package_METS.xml' % (
self.ObjectIdentifierValue)
self.Pmets_objpath = os.path.join(self.AIPpath,
self.Pmets_obj)
self.AIC_UUID = None
self.AIC_UUID_rel_ObjectUUIDs = []
if self.ok:
METS_agent_list = []
METS_altRecordID_list = []
if self.metatype == 1:
############################################
# Object have metatype 1 (METS)
self.METS_LABEL = 'ESSArch AIP'
# Get SIP Content METS information
self.METSfilepath = os.path.join(
self.SIPpath,
self.ObjectIdentifierValue + '/metadata/SIP/' +
self.ObjectIdentifierValue + '_Content_METS.xml')
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.METSfilepath)
for agent in res_info[2]:
if not (agent[0] == 'CREATOR'
and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 2:
############################################
# Object have metatype 2 (RES)
self.METS_LABEL = 'Imaging AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 3:
############################################
# Object have metatype 3 (ADDML)
self.METS_LABEL = 'Born Digital AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype in [4]:
############################################
# Object have metatype 4 (eARD METS)
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.Cmets_objpath)
for agent in res_info[2]:
#if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
self.METS_LABEL = res_info[0][0]
METS_agent_list.append([
'CREATOR', None, 'INDIVIDUAL', None,
AgentIdentifierValue, []
])
METS_agent_list.append([
'CREATOR', None, 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
for altRecordID in res_info[3]:
METS_altRecordID_list.append(altRecordID)
logging.debug('self.obj: ' + str(self.obj))
if self.ChecksumAlgorithm > 0: #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
self.startCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 5)
if errno:
logging.error('Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
logging.info('Start create Package METS for: ' +
self.ObjectIdentifierValue)
if self.ok:
###########################################################
# Create PMETS for AIP package
self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.Cmets_objpath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(
self.Cmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030', '',
'ESSArch AIPCreator',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.M_statinfo = os.stat(self.Cmets_objpath)
self.M_SIZE = self.M_statinfo.st_size
self.M_utc_mtime = datetime.datetime.utcfromtimestamp(
self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.M_lociso_mtime = self.M_utc_mtime.astimezone(
self.tz).isoformat()
self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.ObjectPath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for AIP package: ' + str(
self.ObjectPath)
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.P_statinfo = os.stat(self.ObjectPath)
self.P_SIZE = self.P_statinfo.st_size
self.P_utc_mtime = datetime.datetime.utcfromtimestamp(
self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.P_lociso_mtime = self.P_utc_mtime.astimezone(
self.tz).isoformat()
if self.metatype in [1, 2, 3]:
self.PMETSdoc = ESSMD.createPMets(
ID=self.ObjectIdentifierValue,
LABEL=self.METS_LABEL,
AGENT=METS_agent_list,
P_SIZE=self.P_SIZE,
P_CREATED=self.P_lociso_mtime,
P_CHECKSUM=self.P_CHECKSUM,
P_CHECKSUMTYPE=self.CA,
M_SIZE=self.M_SIZE,
M_CREATED=self.M_lociso_mtime,
M_CHECKSUM=self.M_CHECKSUM,
M_CHECKSUMTYPE=self.CA,
)
errno, why = ESSMD.writeToFile(
self.PMETSdoc, self.Pmets_objpath)
if errno:
self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(
self.Pmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create(
'1040', '', 'ESSArch AIPChecksum',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
time.sleep(2)
self.ok = 0
elif self.metatype in [4]:
ms_files = []
ms_files.append([
'amdSec', None, 'techMD', 'techMD001', None,
None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s/%s' %
(self.ObjectIdentifierValue, self.Cmets_obj),
'simple', self.M_CHECKSUM, self.CA,
self.M_SIZE, 'text/xml', self.M_lociso_mtime,
'OTHER', 'METS', None
])
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % self.p_obj, 'simple',
self.P_CHECKSUM, self.CA, self.P_SIZE,
'application/x-tar', self.P_lociso_mtime,
'tar', 'techMD001', None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.
ObjectIdentifierValue,
METS_ObjectPath=self.Pmets_objpath,
agent_list=METS_agent_list,
altRecordID_list=METS_altRecordID_list,
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL=self.METS_LABEL,
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIP',
METS_DocumentID=self.Pmets_obj,
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create Package METS file, why: %s'
% str(info_list))
self.ObjectMessageDigest = self.P_CHECKSUM
self.stopCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
self.CalTime = self.stopCalTime - self.startCalTime
self.ObjectSizeMB = self.ObjectSize / 1048576
if self.CalTime.seconds < 1:
self.CalTime = datetime.timedelta(
seconds=1
) #Fix min time to 1 second if it is zero.
self.CalMBperSEC = int(self.ObjectSizeMB) / int(
self.CalTime.seconds)
logging.info('Finished calculate checksum: ' +
self.ObjectIdentifierValue + ' , ' +
str(self.CalMBperSEC) + ' MB/Sec and Time: ' +
str(self.CalTime))
if self.ok:
self.timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(
self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue, 'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error(
'Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error(
'Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if self.ok and self.metatype == 4:
####################################################
# Create AIC METS File:
aic_obj = ArchiveObject.objects.filter(
relaic_set__UUID=self.ObjectUUID)[:1]
if aic_obj:
self.AIC_UUID = aic_obj.get().ObjectUUID
logging.info(
'Succeeded to get AIC_UUID: %s from DB' %
self.AIC_UUID)
else:
logging.warning(
'AIC not found for IP object: %s, skip to create AIC METS file'
% self.ObjectUUID)
if self.ok and self.AIC_UUID:
ip_obj_list = ArchiveObject.objects.filter(
Q(StatusProcess=3000)
| Q(ObjectUUID=self.ObjectUUID),
reluuid_set__AIC_UUID=self.AIC_UUID).order_by(
'Generation')
if ip_obj_list:
logging.info('Start create AIC METS: ' +
self.AIC_UUID)
self.AICmets_objpath = os.path.join(
self.AIPpath, self.AIC_UUID + '_AIC_METS.xml')
ms_files = []
for ip_obj in ip_obj_list:
logging.info(
'Add IP: %s to AIC METS: %s' %
(ip_obj.ObjectUUID, self.AIC_UUID))
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % ip_obj.ObjectUUID, 'simple',
ip_obj.ObjectMessageDigest,
dict(ChecksumAlgorithm_CHOICES)[
ip_obj.ObjectMessageDigestAlgorithm],
ip_obj.ObjectSize, 'application/x-tar',
ip_obj.CreateDate, 'IP Package', None, None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.AIC_UUID,
METS_ObjectPath=self.AICmets_objpath,
agent_list=[],
altRecordID_list=[],
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL='AIC relation to IP',
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIC',
METS_DocumentID=self.AIC_UUID +
'_AIC_METS.xml',
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create AIC METS file, why: %s'
% str(info_list))
else:
logging.error(
'Problem to get objects related to AIC_UUID: %s from DB'
% (self.AIC_UUID))
self.ok = 0
if self.ok:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 49, 0)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.info(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
self.event_info, 2,
self.ObjectIdentifierValue)
else:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 100)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
elif self.ChecksumAlgorithm == 0: #self.ChecksumAlgorithm 0 = No checksum
logging.info('Skip creation of checksum: ' +
self.ObjectIdentifierValue)
self.ObjectMessageDigest = ''
self.MetaObjectSize = os.stat(self.Cmets_objpath)[6]
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue,
'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
else:
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
'Skip creation of checksum', 2,
self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag = 0
def ThreadMain(self, ProcName):
logger.info('Starting ' + ProcName)
while 1:
try:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
PauseTime, self.Run = ESSProc.objects.filter(
Name=ProcName).values_list('Time', 'Run')[0]
if self.Run == '0':
logger.info('Stopping ' + ProcName)
ESSProc.objects.filter(Name=ProcName).update(Status='0',
Run='0',
PID=0)
self.RunFlag = 0
self.mLock.release()
if Debug: logger.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
if Debug: logger.info('Start to list worklist')
#######################################
# Start to list robot req
robotQueue_objs = robotQueue.objects.filter(
ReqType__in=[50, 51,
52], Status=0) #Get pending robot requests
for robotQueue_obj in robotQueue_objs:
if ESSProc.objects.get(Name=ProcName).Run == '0':
logger.info('Stopping ' + ProcName)
ESSProc.objects.filter(Name=ProcName).update(
Status='0', Run='0', PID=0)
thread.interrupt_main()
break
t_id = robotQueue_obj.MediumID
if robotQueue_obj.ReqType == 50: #Mount
##########################################
# Check if tape is already mounted
robotdrives_objs = robotdrives.objects.filter(
t_id=t_id, status='Mounted')
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
##########################################
# Tape is mounted, check if locked
if len(robotdrives_obj.drive_lock) > 0:
########################################
# Tape is locked, check if req work_uuid = lock
if robotdrives_obj.drive_lock == robotQueue_obj.ReqUUID:
########################################
# Tape is already locked with req work_uuid
logger.info(
'Already Mounted: ' + str(t_id) +
' and locked by req work_uuid: ' +
str(robotQueue_obj.ReqUUID))
robotQueue_obj.delete()
else:
########################################
# Tape is locked with another work_uuid
logger.info(
'Tape: ' + str(t_id) +
' is busy and locked by: ' +
str(robotdrives_obj.drive_lock) +
' and not req work_uuid: ' +
str(robotQueue_obj.ReqUUID))
else:
########################################
# Tape is not locked, lock the drive with req work_uuid
robotdrives_obj.drive_lock = robotQueue_obj.ReqUUID
robotdrives_obj.save(
update_fields=['drive_lock'])
logger.info(
'Tape: ' + str(t_id) +
' is available set lock to req work_uuid: '
+ str(robotQueue_obj.ReqUUID))
robotQueue_obj.delete()
else:
##########################################
# Tape is not mounted, check for available tape drives
robotdrives_objs = robotdrives.objects.filter(
status='Ready') #Get available tape drives
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
########################################
# Tapedrives is available try to mount tape
robotQueue_obj.Status = 5
robotQueue_obj.save(update_fields=['Status'])
try:
Robot().Mount(t_id,
robotdrives_obj.drive_id,
robotQueue_obj.ReqUUID)
except RobotException as e:
logger.error('Problem to mount tape: ' +
t_id + ' Message: ' + e)
robotQueue_obj.Status = 100
robotQueue_obj.save(
update_fields=['Status'])
else:
#if returncode == 0:
robotQueue_obj.delete()
if storageMedium.objects.filter(
storageMediumID=t_id).exists():
######################################################
# Update StorageMediumTable with num of mounts
storageMedium_obj = storageMedium.objects.get(
storageMediumID=t_id)
timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0,
tzinfo=pytz.utc)
timestamp_dst = timestamp_utc.astimezone(
self.tz)
storageMedium_obj.storageMediumMounts += 1
storageMedium_obj.storageMediumLocationStatus = 50
storageMedium_obj.linkingAgentIdentifierValue = AgentIdentifierValue
storageMedium_obj.LocalDBdatetime = timestamp_utc
storageMedium_obj.save(update_fields=[
'storageMediumMounts',
'storageMediumLocationStatus',
'linkingAgentIdentifierValue',
'LocalDBdatetime'
])
if ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB(
).action(
'storageMedium', 'UPD',
('storageMediumLocationStatus',
50, 'storageMediumMounts',
storageMedium_obj.
storageMediumMounts,
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('storageMediumID', t_id))
if ext_errno:
logger.error(
'Failed to update External DB: '
+ str(t_id) + ' error: ' +
str(ext_why))
else:
storageMedium_obj.ExtDBdatetime = timestamp_utc
storageMedium_obj.save(
update_fields=[
'ExtDBdatetime'
])
#else:
#logger.error('Problem to mount tape: ' + t_id + ' Message: ' + str(mountout))
#robotQueue_obj.Status=100
#robotQueue_obj.save(update_fields=['Status'])
elif robotQueue_obj.ReqType in [
51, 52
]: # Unmount(51), F_Unmount(52)
######################################
# Check if tape is mounted
robotdrives_objs = robotdrives.objects.filter(
t_id=t_id) #Get tape drive that is mounted
if robotdrives_objs:
robotdrives_obj = robotdrives_objs[0]
################################################
# Tape is mounted, check if tape is locked(busy)
if len(robotdrives_obj.drive_lock
) == 0 or robotQueue_obj.ReqType == 52:
if len(robotdrives_obj.drive_lock
) > 0 and robotQueue_obj.ReqType == 52:
logger.info(
'Tape %s is locked with work_uuid %s, try to force unmount',
t_id, robotdrives_obj.drive_lock)
###################################################
# Try to unmount
robotQueue_obj.Status = 5
robotQueue_obj.save(update_fields=['Status'])
robotdrives_obj.status = 'Unmounting'
robotdrives_obj.save(update_fields=['status'])
try:
Robot().Unmount(t_id,
robotdrives_obj.drive_id)
except RobotException as e:
logger.error('Problem to unmount tape: ' +
t_id + ' Message: ' + e)
robotQueue_obj.Status = 100
robotQueue_obj.save(
update_fields=['Status'])
else:
#if returncode == 0:
robotQueue_obj.delete()
#else:
#logger.error('Problem to unmount tape: ' + t_id + ' Message: ' + str(mountout))
#robotQueue_obj.Status=100
#robotQueue_obj.save(update_fields=['Status'])
else:
#################################################
# Tape is locked, skip to unmount
logger.info('Tape ' + t_id +
' is locked, skip to unmount')
else:
################################################
# Tape is not mounted, skip to try to unmount
logger.info('Tape ' + t_id + ' is not mounted')
robotQueue_obj.delete()
db.close_old_connections()
self.mLock.release()
time.sleep(int(PauseTime))
except:
logger.error('Unexpected error: %s' % (str(sys.exc_info())))
raise
self.mDieFlag = 0
ExtDBupdate = int(ESSDB.DB().action('ESSConfig', 'GET', ('Value', ),
('Name', 'ExtDBupdate'))[0][0])
if FTPflag:
self.ftp_res, errno, why = prod().FTPput(
ObjectIdentifierValue=ObjectIdentifierValue,
FileName=FILENAME,
FTPFileName=FTPFileName)
if errno:
return '', 50, 'errno: %s, why: %s' % (str(errno), str(why))
if DBflag:
#print 'DOC',DOC
blob = MySQLdb.escape_string(DOC)
#print 'blobStart#%s#END' % blob
blob_mssql = ESSMSSQL.escape_string(DOC)
#print 'blob_MSSQL2 Start#%s#END' % blob_mssql
if FTPflag and DBflag:
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
ArchiveObject_obj = ArchiveObject.objects.get(
ObjectUUID=ObjectUUID)
ArchiveObjectMetadata_obj = ArchiveObjectMetadata()
ArchiveObjectMetadata_obj.ObjectUUID = ArchiveObject_obj
ArchiveObjectMetadata_obj.ObjectIdentifierValue = ObjectIdentifierValue
ArchiveObjectMetadata_obj.ObjectMetadataType = ObjectMetadataType
ArchiveObjectMetadata_obj.ObjectMetadataServer = self.ftp_res[0]
ArchiveObjectMetadata_obj.ObjectMetadataURL = self.ftp_res[1]
ArchiveObjectMetadata_obj.ObjectMetadataBLOB = blob
def _WriteDiskProc(self, IO_obj_uuid):
"""Writes IP (Information Package) to a disk filepath
:param IO_obj_uuid: Primary key to entry in IOQueue database table
"""
logger = logging.getLogger('StorageMethodDisk')
runflag = 1
contentLocationValue = ''
timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,
tzinfo=pytz.utc)
error_list = []
IO_obj = IOQueue.objects.get(id=IO_obj_uuid)
st_obj = IO_obj.storagemethodtarget
target_obj = st_obj.target
target_obj_target = target_obj.target
sm_obj = st_obj.storagemethod
ArchiveObject_obj = IO_obj.archiveobject
ObjectIdentifierValue = ArchiveObject_obj.ObjectIdentifierValue
source_path = IO_obj.ObjectPath
WriteSize = IO_obj.WriteSize
AgentIdentifierValue = ESSConfig.objects.get(
Name='AgentIdentifierValue').Value
MediumLocation = ESSConfig.objects.get(
Name='storageMediumLocation').Value
ExtDBupdate = int(ESSConfig.objects.get(Name='ExtDBupdate').Value)
storage_obj = None
storageMedium_obj = None
logger.info(
'Start Disk Write Process for object: %s, target: %s, IOuuid: %s',
ObjectIdentifierValue, target_obj_target, IO_obj_uuid)
########################################################
# Check access to ip_tar_path and verify WriteSize
########################################################
aic_obj_uuid = ''
aic_mets_path_source = ''
aic_mets_size = 0
# If storage method format is AIC type (103)
if target_obj.format == 103:
try:
aic_obj_uuid = ArchiveObject_obj.reluuid_set.get().AIC_UUID
except ObjectDoesNotExist as e:
logger.warning(
'Problem to get AIC info for ObjectUUID: %s, error: %s' %
(ObjectIdentifierValue, e))
else:
logger.info('Succeeded to get AIC_UUID: %s from DB' %
aic_obj_uuid)
# Check aic_mets_path_source
aic_mets_path_source = os.path.join(
source_path, '%s_AIC_METS.xml' % aic_obj_uuid)
try:
aic_mets_size = GetSize(aic_mets_path_source)
except OSError as oe:
msg = 'Problem to access AIC METS object: %s, IOuuid: %s, error: %s' % (
aic_mets_path_source, IO_obj_uuid, oe)
logger.error(msg)
error_list.append(msg)
runflag = 0
# Check ip_tar_path_source
ip_tar_filename = '%s.tar' % ArchiveObject_obj.ObjectIdentifierValue
ip_tar_path_source = os.path.join(source_path, ip_tar_filename)
try:
ip_tar_size = GetSize(ip_tar_path_source)
except OSError as oe:
msg = 'Problem to access object: %s, IOuuid: %s, error: %s' % (
ip_tar_path_source, IO_obj_uuid, oe)
logger.error(msg)
error_list.append(msg)
runflag = 0
ip_tar_size = 0
# Check ip_p_mets_path_source
ip_p_mets_path_source = ip_tar_path_source[:-4] + '_Package_METS.xml'
try:
ip_p_mets_size = GetSize(ip_p_mets_path_source)
except OSError as oe:
msg = 'Problem to access metaobject: %s, IOuuid: %s, error: %s' % (
ip_p_mets_path_source, IO_obj_uuid, oe)
logger.error(msg)
error_list.append(msg)
runflag = 0
ip_p_mets_size = 0
# Check WriteSize
if WriteSize:
if not int(WriteSize) == int(ip_tar_size) + int(
ip_p_mets_size) + int(aic_mets_size):
msg = 'Problem defined WriteSize does not match actual filesizes for object: ' + ip_tar_path_source + ', IOuuid: ' + str(
IO_obj_uuid)
logger.error(msg)
error_list.append(msg)
msg = 'WriteSize: ' + str(WriteSize)
logger.error(msg)
error_list.append(msg)
msg = 'ip_tar_size: ' + str(ip_tar_size)
logger.error(msg)
error_list.append(msg)
msg = 'ip_p_mets_size: ' + str(ip_p_mets_size)
logger.error(msg)
error_list.append(msg)
if target_obj.format == 103:
msg = 'aic_mets_size: ' + str(aic_mets_size)
logger.error(msg)
error_list.append(msg)
runflag = 0
else:
WriteSize = int(ip_tar_size) + int(ip_p_mets_size)
if target_obj.format == 103:
WriteSize += int(aic_mets_size)
logger.info(
'WriteSize not defined, setting write size for object: ' +
ObjectIdentifierValue + ' WriteSize: ' + str(WriteSize))
# Check if StorageMediumID 'disk' exist, if exist get current target size.
try:
storageMedium_obj = storageMedium.objects.get(
storageMediumID=target_obj.name)
except storageMedium.DoesNotExist as e:
try:
logger.warning(
'storageMediumID %s not found for IOuuid: %s, try to and new storageMedium'
% (target_obj.name, IO_obj_uuid))
timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
MediumUUID = uuid.uuid4()
storageMedium_obj = storageMedium()
storageMedium_obj.id = MediumUUID
storageMedium_obj.storageMediumUUID = unicode(MediumUUID)
storageMedium_obj.storageMedium = target_obj.type
storageMedium_obj.storageMediumID = target_obj.name
storageMedium_obj.storageMediumDate = timestamp_utc
storageMedium_obj.storageMediumLocation = MediumLocation
storageMedium_obj.storageMediumLocationStatus = 50
storageMedium_obj.storageMediumBlockSize = 128
storageMedium_obj.storageMediumStatus = 20
storageMedium_obj.storageMediumUsedCapacity = 0
storageMedium_obj.storageMediumFormat = target_obj.format
storageMedium_obj.storageMediumMounts = 0
storageMedium_obj.linkingAgentIdentifierValue = AgentIdentifierValue
storageMedium_obj.CreateDate = timestamp_utc
storageMedium_obj.CreateAgentIdentifierValue = AgentIdentifierValue
storageMedium_obj.LocalDBdatetime = timestamp_utc
storageMedium_obj.storagetarget = target_obj
storageMedium_obj.save()
except IntegrityError as e:
if e.args[
0] == 1062: # 1062 = Duplicate entry, try to get object instead
storageMedium_obj = storageMedium.objects.get(
storageMediumID=target_obj.name)
else:
raise e
IO_obj.storagemedium = storageMedium_obj
#new_storageMediumUsedCapacity = storageMedium_obj.storageMediumUsedCapacity + int(WriteSize)
# Check write access to target directory
if not os.access(target_obj_target, 7):
msg = 'Problem to access target directory: %s (IOuuid: %s)' % (
target_obj_target, IO_obj_uuid)
error_list.append(msg)
logger.error(msg)
runflag = 0
startTime = datetime.timedelta(seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
if runflag:
########################################################
# Write AIP package to disk
########################################################
try:
logger.info(
'Try to write %s to storage method disk target: %s, IOuuid: %s',
ObjectIdentifierValue, target_obj_target, IO_obj_uuid)
shutil.copy2(ip_tar_path_source, target_obj_target)
shutil.copy2(ip_p_mets_path_source, target_obj_target)
if target_obj.format == 103:
shutil.copy2(aic_mets_path_source, target_obj_target)
contentLocationValue = target_obj.name
except (IOError, OSError) as e:
msg = 'Problem to write %s to storage method disk target: %s, IOuuid: %s, error: %s' % (
ObjectIdentifierValue, target_obj_target, IO_obj_uuid, e)
logger.error(msg)
error_list.append(msg)
runflag = 0
else:
timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
##########################
# Insert StorageTable
storage_obj = storage()
storage_obj.contentLocationType = sm_obj.type #200
storage_obj.contentLocationValue = contentLocationValue
storage_obj.LocalDBdatetime = timestamp_utc
storage_obj.archiveobject = ArchiveObject_obj
storage_obj.storagemedium = storageMedium_obj
storage_obj.save()
IO_obj.storage = storage_obj
if ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
'storage', 'INS',
('ObjectIdentifierValue',
storage_obj.archiveobject.ObjectIdentifierValue,
'contentLocationType',
storage_obj.contentLocationType,
'contentLocationValue',
storage_obj.contentLocationValue, 'storageMediumID',
storage_obj.storagemedium.storageMediumID))
if ext_errno:
logger.error('Failed to insert to External DB: ' + str(
storage_obj.archiveobject.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
storage_obj.ExtDBdatetime = timestamp_utc
storage_obj.save(update_fields=['ExtDBdatetime'])
if not runflag:
msg = 'Because of the previous problems it is not possible to store the object %s to storage method target: %s, IOuuid: %s' % (
ObjectIdentifierValue, target_obj_target, IO_obj_uuid)
logger.error(msg)
error_list.append(msg)
stopTime = datetime.timedelta(seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
WriteTime = stopTime - startTime
if WriteTime.seconds < 1:
WriteTime = datetime.timedelta(
seconds=1) #Fix min time to 1 second if it is zero.
if storage_obj is not None:
storage_obj_id = storage_obj.id
else:
storage_obj_id = ''
if storageMedium_obj is not None:
storageMedium_obj_storageMediumID = storageMedium_obj.storageMediumID
storageMedium_obj_storageMediumUUID = storageMedium_obj.storageMediumUUID
else:
storageMedium_obj_storageMediumID = ''
storageMedium_obj_storageMediumUUID = ''
res_dict = {
'ObjectIdentifierValue': ArchiveObject_obj.ObjectIdentifierValue,
'ObjectUUID': ArchiveObject_obj.ObjectUUID,
'sm_obj_id': sm_obj.id,
'storage_obj_id': storage_obj_id,
'contentLocationValue': contentLocationValue,
'storageMediumID': storageMedium_obj_storageMediumID,
'storageMediumUUID': storageMedium_obj_storageMediumUUID,
'AgentIdentifierValue': AgentIdentifierValue,
'WriteSize': WriteSize,
'WriteTime': WriteTime,
'timestamp_utc': timestamp_utc,
'error_list': error_list,
'status': runflag,
}
IO_obj.result = res_dict
IO_obj.save(update_fields=['result', 'storagemedium', 'storage'])
if not runflag:
raise ESSArchSMError(error_list)
else:
return res_dict
return '',10,str(why)
elif not DOC:
return '',30,'Missing DOC var'
#self.table = 'IngestObjectMetadata'
ExtDBupdate = int(ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','ExtDBupdate'))[0][0])
if FTPflag:
self.ftp_res,errno,why = prod().FTPput(ObjectIdentifierValue=ObjectIdentifierValue,FileName=FILENAME,FTPFileName=FTPFileName)
if errno: return '',50,'errno: %s, why: %s' % (str(errno),str(why))
if DBflag:
#print 'DOC',DOC
blob = MySQLdb.escape_string(DOC)
#print 'blobStart#%s#END' % blob
blob_mssql = ESSMSSQL.escape_string(DOC)
#print 'blob_MSSQL2 Start#%s#END' % blob_mssql
if FTPflag and DBflag:
self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
ArchiveObject_obj = ArchiveObject.objects.get(ObjectUUID = ObjectUUID)
ArchiveObjectMetadata_obj = ArchiveObjectMetadata()
ArchiveObjectMetadata_obj.ObjectUUID = ArchiveObject_obj
ArchiveObjectMetadata_obj.ObjectIdentifierValue = ObjectIdentifierValue
ArchiveObjectMetadata_obj.ObjectMetadataType = ObjectMetadataType
ArchiveObjectMetadata_obj.ObjectMetadataServer = self.ftp_res[0]
ArchiveObjectMetadata_obj.ObjectMetadataURL = self.ftp_res[1]
ArchiveObjectMetadata_obj.ObjectMetadataBLOB = blob
ArchiveObjectMetadata_obj.linkingAgentIdentifierValue = AgentIdentifierValue
ArchiveObjectMetadata_obj.LocalDBdatetime = self.timestamp_utc.replace(tzinfo=None)