def check_storage():
EL_root = etree.Element('needcopies')
Policy_obj_list = ESSArchPolicy.objects.filter(PolicyStat=1).all()
for Policy_obj in Policy_obj_list:
sm_obj_list = [[Policy_obj.sm_1, Policy_obj.sm_type_1, Policy_obj.sm_format_1, Policy_obj.sm_blocksize_1,
Policy_obj.sm_maxCapacity_1, Policy_obj.sm_minChunkSize_1, Policy_obj.sm_minContainerSize_1,
Policy_obj.sm_target_1],
[Policy_obj.sm_2, Policy_obj.sm_type_2, Policy_obj.sm_format_2, Policy_obj.sm_blocksize_2,
Policy_obj.sm_maxCapacity_2, Policy_obj.sm_minChunkSize_2, Policy_obj.sm_minContainerSize_2,
Policy_obj.sm_target_2],
[Policy_obj.sm_3, Policy_obj.sm_type_3, Policy_obj.sm_format_3, Policy_obj.sm_blocksize_3,
Policy_obj.sm_maxCapacity_3, Policy_obj.sm_minChunkSize_3, Policy_obj.sm_minContainerSize_3,
Policy_obj.sm_target_3],
[Policy_obj.sm_4, Policy_obj.sm_type_4, Policy_obj.sm_format_4, Policy_obj.sm_blocksize_4,
Policy_obj.sm_maxCapacity_4, Policy_obj.sm_minChunkSize_4, Policy_obj.sm_minContainerSize_4,
Policy_obj.sm_target_4],
]
ip_obj_list = ArchiveObject.objects.filter(PolicyId=Policy_obj.PolicyID, StatusProcess=3000, StatusActivity=0).all()
for ip_obj in ip_obj_list:
storage_obj_list = ip_obj.storage_set.all()
sm_num = 0
for sm_obj in sm_obj_list:
sm_num += 1
if sm_obj[0] == 1:
storage_count = 0
storageMediumID_list = []
for storage_obj in storage_obj_list:
storageMedium_obj = storage_obj.storageMediumUUID
if str(sm_obj[1])[0] == '2': #Disk
if storageMedium_obj.storageMedium == sm_obj[1] and storageMedium_obj.storageMediumID == 'disk':
storage_count+=1
storageMediumID_list.append(storageMedium_obj.storageMediumID)
elif str(sm_obj[1])[0] == '3': #Tape
if storageMedium_obj.storageMedium == sm_obj[1] and storageMedium_obj.storageMediumID.startswith(sm_obj[7]):
storage_count+=1
storageMediumID_list.append(storageMedium_obj.storageMediumID)
if storage_count == 0:
EL_object = etree.SubElement(EL_root, 'object', attrib={'id':ip_obj.ObjectIdentifierValue,
'target':sm_obj[7],
})
print 'Missing storage entry for storage method number: %s, target: %s, for object: %s' % ( sm_num, sm_obj[7], ip_obj.ObjectIdentifierValue)
elif storage_count == 1:
print 'Found storage entry for storage method number: %s, target: %s (%s), for object: %s' % ( sm_num, sm_obj[7], ','.join(str(e) for e in storageMediumID_list), ip_obj.ObjectIdentifierValue)
else:
print 'Warning found to many storage entry for storage method number: %s, target: %s (%s), for object: %s' % ( sm_num, sm_obj[7], ','.join(str(e) for e in storageMediumID_list), ip_obj.ObjectIdentifierValue)
doc = etree.ElementTree(element=EL_root, file=None)
ESSMD.writeToFile(doc,'/ESSArch/log/needcopies/needcopies.xml')
def ThreadMain(self,ProcName):
logging.info('Starting ' + ProcName)
TimeZone = timezone.get_default_timezone_name()
self.tz=pytz.timezone(TimeZone)
METS_NAMESPACE = SchemaProfile.objects.get(entity = 'mets_namespace').value
METS_SCHEMALOCATION = SchemaProfile.objects.get(entity = 'mets_schemalocation').value
METS_PROFILE = SchemaProfile.objects.get(entity = 'mets_profile').value
XLINK_NAMESPACE = SchemaProfile.objects.get(entity = 'xlink_namespace').value
XSI_NAMESPACE = SchemaProfile.objects.get(entity = 'xsi_namespace').value
while 1:
if self.mDieFlag==1: break # Request for death
self.mLock.acquire()
self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
self.RunFlag=0
self.mLock.release()
if Debug: logging.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock=thread.allocate_lock()
Cmets_obj = Parameter.objects.get(entity='content_descriptionfile').value
self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue',
'ObjectUUID',
'PolicyId',
'ObjectSize'),
('StatusProcess','BETWEEN',39,'AND',40,'AND',
'StatusActivity','=','0'))
if errno: logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ok = 1
self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
if self.ProcDB[0]=='0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1]==1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
if self.ProcDB[1]==1:
logging.info('Process is in pause state')
else:
break
self.ObjectIdentifierValue = self.obj[0]
self.ObjectUUID = self.obj[1]
self.PolicyId = self.obj[2]
self.ObjectSize = self.obj[3]
self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','ChecksumAlgorithm','IngestPath'),('PolicyID',self.PolicyId))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
self.ok = 0
if self.ok:
###########################################################
# set variables
self.AIPpath = self.PolicyDB[0][0]
self.metatype = self.PolicyDB[0][1]
self.ChecksumAlgorithm = self.PolicyDB[0][2]
self.CA = dict(ChecksumAlgorithm_CHOICES)[self.ChecksumAlgorithm]
self.SIPpath = self.PolicyDB[0][3]
self.p_obj = self.ObjectIdentifierValue + '.tar'
self.ObjectPath = os.path.join(self.AIPpath,self.p_obj)
self.SIProotpath = os.path.join(self.SIPpath,self.ObjectIdentifierValue)
if self.metatype in [4]:
#self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
#self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
#self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
self.Cmets_obj = Cmets_obj.replace('{objid}',self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(self.SIProotpath,self.Cmets_obj)
elif self.metatype in [1,2,3]:
self.Cmets_obj = '%s_Content_METS.xml' % (self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(self.AIPpath,self.Cmets_obj)
self.Pmets_obj = '%s_Package_METS.xml' % (self.ObjectIdentifierValue)
self.Pmets_objpath = os.path.join(self.AIPpath,self.Pmets_obj)
self.AIC_UUID = None
self.AIC_UUID_rel_ObjectUUIDs = []
if self.ok:
METS_agent_list = []
METS_altRecordID_list = []
if self.metatype == 1:
############################################
# Object have metatype 1 (METS)
self.METS_LABEL = 'ESSArch AIP'
# Get SIP Content METS information
self.METSfilepath = os.path.join(self.SIPpath,self.ObjectIdentifierValue + '/metadata/SIP/' + self.ObjectIdentifierValue + '_Content_METS.xml')
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.METSfilepath)
for agent in res_info[2]:
if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
elif self.metatype == 2:
############################################
# Object have metatype 2 (RES)
self.METS_LABEL = 'Imaging AIP RA'
METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
elif self.metatype == 3:
############################################
# Object have metatype 3 (ADDML)
self.METS_LABEL = 'Born Digital AIP RA'
METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
elif self.metatype in [4]:
############################################
# Object have metatype 4 (eARD METS)
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.Cmets_objpath)
for agent in res_info[2]:
#if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
self.METS_LABEL = res_info[0][0]
METS_agent_list.append(['CREATOR',None, 'INDIVIDUAL',None,AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR',None, 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
for altRecordID in res_info[3]:
METS_altRecordID_list.append(altRecordID)
logging.debug('self.obj: '+str(self.obj))
if self.ChecksumAlgorithm > 0: #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
self.startCalTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 40, 5)
if errno: logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
logging.info('Start create Package METS for: ' + self.ObjectIdentifierValue)
if self.ok:
###########################################################
# Create PMETS for AIP package
self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(self.Cmets_objpath,self.CA)
if errno:
self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(self.Cmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
self.M_statinfo = os.stat(self.Cmets_objpath)
self.M_SIZE = self.M_statinfo.st_size
self.M_utc_mtime = datetime.datetime.utcfromtimestamp(self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.M_lociso_mtime = self.M_utc_mtime.astimezone(self.tz).isoformat()
self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(self.ObjectPath,self.CA)
if errno:
self.event_info = 'Problem to get checksum for AIP package: ' + str(self.ObjectPath)
logging.error(self.event_info)
ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
self.P_statinfo = os.stat(self.ObjectPath)
self.P_SIZE = self.P_statinfo.st_size
self.P_utc_mtime = datetime.datetime.utcfromtimestamp(self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.P_lociso_mtime = self.P_utc_mtime.astimezone(self.tz).isoformat()
if self.metatype in [1,2,3]:
self.PMETSdoc = ESSMD.createPMets(
ID=self.ObjectIdentifierValue,
LABEL=self.METS_LABEL,
AGENT=METS_agent_list,
P_SIZE=self.P_SIZE,
P_CREATED=self.P_lociso_mtime,
P_CHECKSUM=self.P_CHECKSUM,
P_CHECKSUMTYPE=self.CA,
M_SIZE=self.M_SIZE,
M_CREATED=self.M_lociso_mtime,
M_CHECKSUM=self.M_CHECKSUM,
M_CHECKSUMTYPE=self.CA,
)
errno,why = ESSMD.writeToFile(self.PMETSdoc,self.Pmets_objpath)
if errno:
self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(self.Pmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
time.sleep(2)
self.ok = 0
elif self.metatype in [4]:
ms_files = []
ms_files.append(['amdSec', None, 'techMD', 'techMD001', None,
None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s/%s' % (self.ObjectIdentifierValue,self.Cmets_obj), 'simple',
self.M_CHECKSUM, self.CA, self.M_SIZE, 'text/xml', self.M_lociso_mtime,
'OTHER', 'METS', None])
ms_files.append(['fileSec', None, None, None, None,
None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s' % self.p_obj, 'simple',
self.P_CHECKSUM, self.CA, self.P_SIZE, 'application/x-tar', self.P_lociso_mtime,
'tar', 'techMD001', None])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (METS_NAMESPACE, METS_SCHEMALOCATION)
errno,info_list = ESSMD.Create_IP_mets(ObjectIdentifierValue = self.ObjectIdentifierValue,
METS_ObjectPath = self.Pmets_objpath,
agent_list = METS_agent_list,
altRecordID_list = METS_altRecordID_list,
file_list = ms_files,
namespacedef = self.namespacedef,
METS_LABEL = self.METS_LABEL,
METS_PROFILE = METS_PROFILE,
METS_TYPE = 'AIP',
METS_DocumentID = self.Pmets_obj,
TimeZone = TimeZone)
if errno:
logging.error('Problem to create Package METS file, why: %s' % str(info_list))
self.ObjectMessageDigest = self.P_CHECKSUM
self.stopCalTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
self.CalTime = self.stopCalTime-self.startCalTime
self.ObjectSizeMB = self.ObjectSize/1048576
if self.CalTime.seconds < 1: self.CalTime = datetime.timedelta(seconds=1) #Fix min time to 1 second if it is zero.
self.CalMBperSEC = int(self.ObjectSizeMB)/int(self.CalTime.seconds)
logging.info('Finished calculate checksum: ' + self.ObjectIdentifierValue + ' , ' + str(self.CalMBperSEC) + ' MB/Sec and Time: ' + str(self.CalTime))
if self.ok:
self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
'ObjectMessageDigest',self.ObjectMessageDigest,
'MetaObjectSize',self.M_SIZE,
'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue,
'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
if errno == 0 and ExtDBupdate:
ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
'ObjectMessageDigest',self.ObjectMessageDigest,
'MetaObjectSize',self.M_SIZE,
'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
else:
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
if self.ok and self.metatype == 4:
####################################################
# Create AIC METS File:
aic_obj = ArchiveObject.objects.filter(relaic_set__UUID=self.ObjectUUID)[:1]
if aic_obj:
self.AIC_UUID = aic_obj.get().ObjectUUID
logging.info('Succeeded to get AIC_UUID: %s from DB' % self.AIC_UUID)
else:
logging.warning('AIC not found for IP object: %s, skip to create AIC METS file' % self.ObjectUUID)
if self.ok and self.AIC_UUID:
ip_obj_list = ArchiveObject.objects.filter(Q(StatusProcess=3000) | Q(ObjectUUID=self.ObjectUUID), reluuid_set__AIC_UUID=self.AIC_UUID).order_by('Generation')
if ip_obj_list:
logging.info('Start create AIC METS: ' + self.AIC_UUID)
self.AICmets_objpath = os.path.join(self.AIPpath,self.AIC_UUID + '_AIC_METS.xml')
ms_files = []
for ip_obj in ip_obj_list:
logging.info('Add IP: %s to AIC METS: %s' % (ip_obj.ObjectUUID,self.AIC_UUID))
ms_files.append(['fileSec', None, None, None, None,
None, 'ID%s' % str(uuid.uuid1()), 'URL', 'file:%s' % ip_obj.ObjectUUID, 'simple',
ip_obj.ObjectMessageDigest, dict(ChecksumAlgorithm_CHOICES)[ip_obj.ObjectMessageDigestAlgorithm], ip_obj.ObjectSize, 'application/x-tar', ip_obj.CreateDate,
'IP Package', None, None])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (METS_NAMESPACE, METS_SCHEMALOCATION)
errno,info_list = ESSMD.Create_IP_mets(ObjectIdentifierValue = self.AIC_UUID,
METS_ObjectPath = self.AICmets_objpath,
agent_list = [],
altRecordID_list = [],
file_list = ms_files,
namespacedef = self.namespacedef,
METS_LABEL = 'AIC relation to IP',
METS_PROFILE = METS_PROFILE,
METS_TYPE = 'AIC',
METS_DocumentID = self.AIC_UUID + '_AIC_METS.xml',
TimeZone = TimeZone)
if errno:
logging.error('Problem to create AIC METS file, why: %s' % str(info_list))
else:
logging.error('Problem to get objects related to AIC_UUID: %s from DB' % (self.AIC_UUID))
self.ok = 0
if self.ok:
errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 49, 0)
if errno:
logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else:
self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.info(self.event_info)
ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'0',self.event_info,2,self.ObjectIdentifierValue)
else:
errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 40, 100)
if errno:
logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else:
self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.error(self.event_info)
ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
elif self.ChecksumAlgorithm == 0: #self.ChecksumAlgorithm 0 = No checksum
logging.info('Skip creation of checksum: ' + self.ObjectIdentifierValue)
self.ObjectMessageDigest = ''
self.MetaObjectSize = os.stat(self.Cmets_objpath)[6]
self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
'ObjectMessageDigest',self.ObjectMessageDigest,
'StatusProcess','49',
'StatusActivity','0',
'MetaObjectSize',self.MetaObjectSize,
'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue,
'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else: ESSPGM.Events().create('1040','','ESSArch AIPChecksum',ProcVersion,'0','Skip creation of checksum',2,self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.IngestTable,'UPD',('ObjectMessageDigestAlgorithm',self.ChecksumAlgorithm,
'ObjectMessageDigest',self.ObjectMessageDigest,
'StatusProcess','49',
'StatusActivity','0',
'MetaObjectSize',self.MetaObjectSize,
'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
else:
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag=0
def ThreadMain(self, ProcName):
logging.info('Starting ' + ProcName)
TimeZone = timezone.get_default_timezone_name()
self.tz = pytz.timezone(TimeZone)
METS_NAMESPACE = SchemaProfile.objects.get(
entity='mets_namespace').value
METS_SCHEMALOCATION = SchemaProfile.objects.get(
entity='mets_schemalocation').value
METS_PROFILE = SchemaProfile.objects.get(entity='mets_profile').value
XLINK_NAMESPACE = SchemaProfile.objects.get(
entity='xlink_namespace').value
XSI_NAMESPACE = SchemaProfile.objects.get(entity='xsi_namespace').value
while 1:
if self.mDieFlag == 1: break # Request for death
self.mLock.acquire()
self.Time, self.Run = ESSDB.DB().action('ESSProc', 'GET',
('Time', 'Run'),
('Name', ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
self.RunFlag = 0
self.mLock.release()
if Debug: logging.info('RunFlag: 0')
time.sleep(2)
continue
# Process Item
lock = thread.allocate_lock()
Cmets_obj = Parameter.objects.get(
entity='content_descriptionfile').value
self.IngestTable = ESSDB.DB().action('ESSConfig', 'GET',
('Value', ),
('Name', 'IngestTable'))[0][0]
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
self.dbget, errno, why = ESSDB.DB().action(
self.IngestTable, 'GET4',
('ObjectIdentifierValue', 'ObjectUUID', 'PolicyId',
'ObjectSize'), ('StatusProcess', 'BETWEEN', 39, 'AND', 40,
'AND', 'StatusActivity', '=', '0'))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ok = 1
self.ProcDB = ESSDB.DB().action('ESSProc', 'GET',
('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[0] == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc', 'UPD',
('Status', '0', 'Run', '0', 'PID', '0'),
('Name', ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1] == 1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action(
'ESSProc', 'GET', ('Run', 'Pause'),
('Name', ProcName))[0]
if self.ProcDB[1] == 1:
logging.info('Process is in pause state')
else:
break
self.ObjectIdentifierValue = self.obj[0]
self.ObjectUUID = self.obj[1]
self.PolicyId = self.obj[2]
self.ObjectSize = self.obj[3]
ArchivePolicy_obj = ArchivePolicy.objects.get(
PolicyStat=1, PolicyID=self.PolicyId)
if self.ok:
###########################################################
# set variables
self.AIPpath = ArchivePolicy_obj.AIPpath
self.metatype = ArchivePolicy_obj.IngestMetadata
self.ChecksumAlgorithm = ArchivePolicy_obj.ChecksumAlgorithm
self.CA = dict(ChecksumAlgorithm_CHOICES)[
self.ChecksumAlgorithm]
self.SIPpath = ArchivePolicy_obj.IngestPath
self.p_obj = self.ObjectIdentifierValue + '.tar'
self.ObjectPath = os.path.join(self.AIPpath, self.p_obj)
self.SIProotpath = os.path.join(self.SIPpath,
self.ObjectIdentifierValue)
if self.metatype in [4]:
#self.Cmets_obj = '%s/%s_Content_METS.xml' % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
#self.Cmets_objpath = os.path.join(self.SIPpath,self.Cmets_obj)
#self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
self.Cmets_obj = Cmets_obj.replace(
'{objid}', self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.SIProotpath, self.Cmets_obj)
elif self.metatype in [1, 2, 3]:
self.Cmets_obj = '%s_Content_METS.xml' % (
self.ObjectIdentifierValue)
self.Cmets_objpath = os.path.join(
self.AIPpath, self.Cmets_obj)
self.Pmets_obj = '%s_Package_METS.xml' % (
self.ObjectIdentifierValue)
self.Pmets_objpath = os.path.join(self.AIPpath,
self.Pmets_obj)
self.AIC_UUID = None
self.AIC_UUID_rel_ObjectUUIDs = []
if self.ok:
METS_agent_list = []
METS_altRecordID_list = []
if self.metatype == 1:
############################################
# Object have metatype 1 (METS)
self.METS_LABEL = 'ESSArch AIP'
# Get SIP Content METS information
self.METSfilepath = os.path.join(
self.SIPpath,
self.ObjectIdentifierValue + '/metadata/SIP/' +
self.ObjectIdentifierValue + '_Content_METS.xml')
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.METSfilepath)
for agent in res_info[2]:
if not (agent[0] == 'CREATOR'
and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 2:
############################################
# Object have metatype 2 (RES)
self.METS_LABEL = 'Imaging AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype == 3:
############################################
# Object have metatype 3 (ADDML)
self.METS_LABEL = 'Born Digital AIP RA'
METS_agent_list.append([
'ARCHIVIST', 'ORGANIZATION', '', 'Riksarkivet', []
])
METS_agent_list.append(
['CREATOR', 'ORGANIZATION', '', 'Riksarkivet', []])
METS_agent_list.append([
'CREATOR', 'INDIVIDUAL', '', AgentIdentifierValue,
[]
])
METS_agent_list.append([
'CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
elif self.metatype in [4]:
############################################
# Object have metatype 4 (eARD METS)
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(
FILENAME=self.Cmets_objpath)
for agent in res_info[2]:
#if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
self.METS_LABEL = res_info[0][0]
METS_agent_list.append([
'CREATOR', None, 'INDIVIDUAL', None,
AgentIdentifierValue, []
])
METS_agent_list.append([
'CREATOR', None, 'OTHER', 'SOFTWARE', 'ESSArch',
['VERSION=%s' % ProcVersion]
])
for altRecordID in res_info[3]:
METS_altRecordID_list.append(altRecordID)
logging.debug('self.obj: ' + str(self.obj))
if self.ChecksumAlgorithm > 0: #self.ChecksumAlgorithm 1 = MD5, 2 = SHA-256
self.startCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 5)
if errno:
logging.error('Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
logging.info('Start create Package METS for: ' +
self.ObjectIdentifierValue)
if self.ok:
###########################################################
# Create PMETS for AIP package
self.M_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.Cmets_objpath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for METS object for AIP package: ' + str(
self.Cmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030', '',
'ESSArch AIPCreator',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.M_statinfo = os.stat(self.Cmets_objpath)
self.M_SIZE = self.M_statinfo.st_size
self.M_utc_mtime = datetime.datetime.utcfromtimestamp(
self.M_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.M_lociso_mtime = self.M_utc_mtime.astimezone(
self.tz).isoformat()
self.P_CHECKSUM, errno, why = ESSPGM.Check().checksum(
self.ObjectPath, self.CA)
if errno:
self.event_info = 'Problem to get checksum for AIP package: ' + str(
self.ObjectPath)
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
self.ok = 0
self.P_statinfo = os.stat(self.ObjectPath)
self.P_SIZE = self.P_statinfo.st_size
self.P_utc_mtime = datetime.datetime.utcfromtimestamp(
self.P_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.P_lociso_mtime = self.P_utc_mtime.astimezone(
self.tz).isoformat()
if self.metatype in [1, 2, 3]:
self.PMETSdoc = ESSMD.createPMets(
ID=self.ObjectIdentifierValue,
LABEL=self.METS_LABEL,
AGENT=METS_agent_list,
P_SIZE=self.P_SIZE,
P_CREATED=self.P_lociso_mtime,
P_CHECKSUM=self.P_CHECKSUM,
P_CHECKSUMTYPE=self.CA,
M_SIZE=self.M_SIZE,
M_CREATED=self.M_lociso_mtime,
M_CHECKSUM=self.M_CHECKSUM,
M_CHECKSUMTYPE=self.CA,
)
errno, why = ESSMD.writeToFile(
self.PMETSdoc, self.Pmets_objpath)
if errno:
self.event_info = 'Problem to write PMETS to file for AIP package: ' + str(
self.Pmets_objpath)
logging.error(self.event_info)
ESSPGM.Events().create(
'1040', '', 'ESSArch AIPChecksum',
ProcVersion, '1', self.event_info, 2,
self.ObjectIdentifierValue)
time.sleep(2)
self.ok = 0
elif self.metatype in [4]:
ms_files = []
ms_files.append([
'amdSec', None, 'techMD', 'techMD001', None,
None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s/%s' %
(self.ObjectIdentifierValue, self.Cmets_obj),
'simple', self.M_CHECKSUM, self.CA,
self.M_SIZE, 'text/xml', self.M_lociso_mtime,
'OTHER', 'METS', None
])
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % self.p_obj, 'simple',
self.P_CHECKSUM, self.CA, self.P_SIZE,
'application/x-tar', self.P_lociso_mtime,
'tar', 'techMD001', None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.
ObjectIdentifierValue,
METS_ObjectPath=self.Pmets_objpath,
agent_list=METS_agent_list,
altRecordID_list=METS_altRecordID_list,
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL=self.METS_LABEL,
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIP',
METS_DocumentID=self.Pmets_obj,
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create Package METS file, why: %s'
% str(info_list))
self.ObjectMessageDigest = self.P_CHECKSUM
self.stopCalTime = datetime.timedelta(
seconds=time.localtime()[5],
minutes=time.localtime()[4],
hours=time.localtime()[3])
self.CalTime = self.stopCalTime - self.startCalTime
self.ObjectSizeMB = self.ObjectSize / 1048576
if self.CalTime.seconds < 1:
self.CalTime = datetime.timedelta(
seconds=1
) #Fix min time to 1 second if it is zero.
self.CalMBperSEC = int(self.ObjectSizeMB) / int(
self.CalTime.seconds)
logging.info('Finished calculate checksum: ' +
self.ObjectIdentifierValue + ' , ' +
str(self.CalMBperSEC) + ' MB/Sec and Time: ' +
str(self.CalTime))
if self.ok:
self.timestamp_utc = datetime.datetime.utcnow(
).replace(microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(
self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue, 'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'MetaObjectSize',
self.M_SIZE, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error(
'Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error(
'Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
if self.ok and self.metatype == 4:
####################################################
# Create AIC METS File:
aic_obj = ArchiveObject.objects.filter(
relaic_set__UUID=self.ObjectUUID)[:1]
if aic_obj:
self.AIC_UUID = aic_obj.get().ObjectUUID
logging.info(
'Succeeded to get AIC_UUID: %s from DB' %
self.AIC_UUID)
else:
logging.warning(
'AIC not found for IP object: %s, skip to create AIC METS file'
% self.ObjectUUID)
if self.ok and self.AIC_UUID:
ip_obj_list = ArchiveObject.objects.filter(
Q(StatusProcess=3000)
| Q(ObjectUUID=self.ObjectUUID),
reluuid_set__AIC_UUID=self.AIC_UUID).order_by(
'Generation')
if ip_obj_list:
logging.info('Start create AIC METS: ' +
self.AIC_UUID)
self.AICmets_objpath = os.path.join(
self.AIPpath, self.AIC_UUID + '_AIC_METS.xml')
ms_files = []
for ip_obj in ip_obj_list:
logging.info(
'Add IP: %s to AIC METS: %s' %
(ip_obj.ObjectUUID, self.AIC_UUID))
ms_files.append([
'fileSec', None, None, None, None, None,
'ID%s' % str(uuid.uuid1()), 'URL',
'file:%s' % ip_obj.ObjectUUID, 'simple',
ip_obj.ObjectMessageDigest,
dict(ChecksumAlgorithm_CHOICES)[
ip_obj.ObjectMessageDigestAlgorithm],
ip_obj.ObjectSize, 'application/x-tar',
ip_obj.CreateDate, 'IP Package', None, None
])
# define namespaces
self.namespacedef = 'xmlns:mets="%s"' % METS_NAMESPACE
self.namespacedef += ' xmlns:xlink="%s"' % XLINK_NAMESPACE
self.namespacedef += ' xmlns:xsi="%s"' % XSI_NAMESPACE
self.namespacedef += ' xsi:schemaLocation="%s %s"' % (
METS_NAMESPACE, METS_SCHEMALOCATION)
errno, info_list = ESSMD.Create_IP_mets(
ObjectIdentifierValue=self.AIC_UUID,
METS_ObjectPath=self.AICmets_objpath,
agent_list=[],
altRecordID_list=[],
file_list=ms_files,
namespacedef=self.namespacedef,
METS_LABEL='AIC relation to IP',
METS_PROFILE=METS_PROFILE,
METS_TYPE='AIC',
METS_DocumentID=self.AIC_UUID +
'_AIC_METS.xml',
TimeZone=TimeZone)
if errno:
logging.error(
'Problem to create AIC METS file, why: %s'
% str(info_list))
else:
logging.error(
'Problem to get objects related to AIC_UUID: %s from DB'
% (self.AIC_UUID))
self.ok = 0
if self.ok:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 49, 0)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Succeeded to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.info(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
self.event_info, 2,
self.ObjectIdentifierValue)
else:
errno, why = ESSPGM.DB().SetAIPstatus(
self.IngestTable, self.ext_IngestTable,
AgentIdentifierValue, self.ObjectUUID, 40, 100)
if errno:
logging.error(
'Failed to update DB status for AIP: ' +
str(self.ObjectIdentifierValue) + ' error: ' +
str(why))
else:
self.event_info = 'Failed to create checksum for Object: %s' % self.ObjectIdentifierValue
logging.error(self.event_info)
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '1',
self.event_info, 2,
self.ObjectIdentifierValue)
elif self.ChecksumAlgorithm == 0: #self.ChecksumAlgorithm 0 = No checksum
logging.info('Skip creation of checksum: ' +
self.ObjectIdentifierValue)
self.ObjectMessageDigest = ''
self.MetaObjectSize = os.stat(self.Cmets_objpath)[6]
self.timestamp_utc = datetime.datetime.utcnow().replace(
microsecond=0, tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue', AgentIdentifierValue,
'LocalDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue', self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
else:
ESSPGM.Events().create('1040', '',
'ESSArch AIPChecksum',
ProcVersion, '0',
'Skip creation of checksum', 2,
self.ObjectIdentifierValue)
if errno == 0 and ExtDBupdate:
ext_res, ext_errno, ext_why = ESSMSSQL.DB().action(
self.IngestTable, 'UPD',
('ObjectMessageDigestAlgorithm',
self.ChecksumAlgorithm, 'ObjectMessageDigest',
self.ObjectMessageDigest, 'StatusProcess', '49',
'StatusActivity', '0', 'MetaObjectSize',
self.MetaObjectSize, 'LastEventDate',
self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',
AgentIdentifierValue),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if ext_errno:
logging.error('Failed to update External DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(ext_why))
else:
res, errno, why = ESSDB.DB().action(
self.IngestTable, 'UPD',
('ExtDBdatetime',
self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',
self.ObjectIdentifierValue))
if errno:
logging.error('Failed to update Local DB: ' +
str(self.ObjectIdentifierValue) +
' error: ' + str(why))
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag = 0
def check_storage():
EL_root = etree.Element('needcopies')
Policy_obj_list = ESSArchPolicy.objects.filter(PolicyStat=1).all()
for Policy_obj in Policy_obj_list:
sm_obj_list = [
[
Policy_obj.sm_1, Policy_obj.sm_type_1, Policy_obj.sm_format_1,
Policy_obj.sm_blocksize_1, Policy_obj.sm_maxCapacity_1,
Policy_obj.sm_minChunkSize_1, Policy_obj.sm_minContainerSize_1,
Policy_obj.sm_target_1
],
[
Policy_obj.sm_2, Policy_obj.sm_type_2, Policy_obj.sm_format_2,
Policy_obj.sm_blocksize_2, Policy_obj.sm_maxCapacity_2,
Policy_obj.sm_minChunkSize_2, Policy_obj.sm_minContainerSize_2,
Policy_obj.sm_target_2
],
[
Policy_obj.sm_3, Policy_obj.sm_type_3, Policy_obj.sm_format_3,
Policy_obj.sm_blocksize_3, Policy_obj.sm_maxCapacity_3,
Policy_obj.sm_minChunkSize_3, Policy_obj.sm_minContainerSize_3,
Policy_obj.sm_target_3
],
[
Policy_obj.sm_4, Policy_obj.sm_type_4, Policy_obj.sm_format_4,
Policy_obj.sm_blocksize_4, Policy_obj.sm_maxCapacity_4,
Policy_obj.sm_minChunkSize_4, Policy_obj.sm_minContainerSize_4,
Policy_obj.sm_target_4
],
]
ip_obj_list = ArchiveObject.objects.filter(
PolicyId=Policy_obj.PolicyID, StatusProcess=3000,
StatusActivity=0).all()
for ip_obj in ip_obj_list:
storage_obj_list = ip_obj.storage_set.all()
sm_num = 0
for sm_obj in sm_obj_list:
sm_num += 1
if sm_obj[0] == 1:
storage_count = 0
storageMediumID_list = []
for storage_obj in storage_obj_list:
storageMedium_obj = storage_obj.storageMediumUUID
if str(sm_obj[1])[0] == '2': #Disk
if storageMedium_obj.storageMedium == sm_obj[
1] and storageMedium_obj.storageMediumID == 'disk':
storage_count += 1
storageMediumID_list.append(
storageMedium_obj.storageMediumID)
elif str(sm_obj[1])[0] == '3': #Tape
if storageMedium_obj.storageMedium == sm_obj[
1] and storageMedium_obj.storageMediumID.startswith(
sm_obj[7]):
storage_count += 1
storageMediumID_list.append(
storageMedium_obj.storageMediumID)
if storage_count == 0:
EL_object = etree.SubElement(
EL_root,
'object',
attrib={
'id': ip_obj.ObjectIdentifierValue,
'target': sm_obj[7],
})
print 'Missing storage entry for storage method number: %s, target: %s, for object: %s' % (
sm_num, sm_obj[7], ip_obj.ObjectIdentifierValue)
elif storage_count == 1:
print 'Found storage entry for storage method number: %s, target: %s (%s), for object: %s' % (
sm_num, sm_obj[7], ','.join(
str(e) for e in storageMediumID_list),
ip_obj.ObjectIdentifierValue)
else:
print 'Warning found to many storage entry for storage method number: %s, target: %s (%s), for object: %s' % (
sm_num, sm_obj[7], ','.join(
str(e) for e in storageMediumID_list),
ip_obj.ObjectIdentifierValue)
doc = etree.ElementTree(element=EL_root, file=None)
ESSMD.writeToFile(doc, '/ESSArch/log/needcopies/needcopies.xml')
def ThreadMain(self,ProcName):
logging.info('Starting ' + ProcName)
self.tz=timezone.get_default_timezone()
while 1:
if self.mDieFlag==1: break # Request for death
self.mLock.acquire()
self.Time,self.Run = ESSDB.DB().action('ESSProc','GET',('Time','Run'),('Name',ProcName))[0]
if self.Run == '0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
self.RunFlag=0
self.mLock.release()
#if Debug: print 'RunFlag: 0'
time.sleep(2)
continue
# Process Item
lock=thread.allocate_lock()
self.IngestTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','IngestTable'))[0][0]
self.PolicyTable = ESSDB.DB().action('ESSConfig','GET',('Value',),('Name','PolicyTable'))[0][0]
Cmets_obj = Parameter.objects.get(entity='content_descriptionfile').value
if ExtDBupdate:
self.ext_IngestTable = self.IngestTable
else:
self.ext_IngestTable = ''
self.dbget,errno,why = ESSDB.DB().action(self.IngestTable,'GET4',('ObjectIdentifierValue','ObjectUUID','PolicyId'),('StatusProcess','BETWEEN',29,'AND',31,'AND',
'StatusActivity','=','0'))
if errno: logging.error('Failed to access Local DB, error: ' + str(why))
for self.obj in self.dbget:
self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
if self.ProcDB[0]=='0':
logging.info('Stopping ' + ProcName)
ESSDB.DB().action('ESSProc','UPD',('Status','0','Run','0','PID','0'),('Name',ProcName))
thread.interrupt_main()
time.sleep(5)
break
elif self.ProcDB[1]==1:
while 1:
time.sleep(60)
self.ProcDB = ESSDB.DB().action('ESSProc','GET',('Run','Pause'),('Name',ProcName))[0]
if self.ProcDB[1]==1:
logging.info('Process is in pause state')
else:
break
self.ok = 1
###########################################################
# get policy info
self.ObjectIdentifierValue = self.obj[0]
self.ObjectUUID = self.obj[1]
self.PolicyId = self.obj[2]
logging.info('Start to create AIP for: %s', self.ObjectIdentifierValue)
self.PolicyDB,errno,why = ESSDB.DB().action(self.PolicyTable,'GET3',('AIPpath','IngestMetadata','IngestPath'),('PolicyID',self.PolicyId))
if errno:
logging.error('Failed to access Local DB, error: ' + str(why))
self.ok = 0
if self.ok:
###########################################################
# set variables
self.AIPpath = self.PolicyDB[0][0]
self.metatype = self.PolicyDB[0][1]
self.SIPpath = self.PolicyDB[0][2]
self.p_obj = self.ObjectIdentifierValue + '.tar'
self.p_objpath = os.path.join(self.AIPpath,self.p_obj)
#self.Cmets_obj = self.ObjectIdentifierValue + '_Content_METS.xml'
#self.Cmets_obj = Cmets_obj.replace('{uuid}',self.ObjectIdentifierValue)
self.Cmets_obj = Cmets_obj.replace('{objid}',self.ObjectIdentifierValue)
self.SIProotpath = os.path.join(self.SIPpath,self.ObjectIdentifierValue)
if self.metatype in [4]:
self.Cmets_objpath = os.path.join(self.SIProotpath,self.Cmets_obj)
if os.path.exists(os.path.join(self.SIProotpath,'sip.xml')):
mets_file = 'sip.xml'
self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
elif os.path.exists(os.path.join(self.SIProotpath,'mets.xml')):
mets_file = 'mets.xml'
self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
#elif os.path.exists(os.path.join(self.SIProotpath,'%s_Content_METS.xml' % self.ObjectIdentifierValue)):
# mets_file = '%s_Content_METS.xml' % self.ObjectIdentifierValue
# self.SIPmets_objpath = os.path.join(self.SIProotpath,mets_file)
else:
self.SIPmets_objpath = ''
elif self.metatype in [1,2,3]:
self.Cmets_objpath = os.path.join(self.AIPpath,self.Cmets_obj)
Debug = 1
logging.debug('self.obj: %s', str(self.obj))
logging.debug('self.ObjectIdentifierValue: %s', self.ObjectIdentifierValue)
logging.debug('Len self.ObjectIdentifierValue: %s', len(self.ObjectIdentifierValue))
logging.debug('self.SIPpath: %s', self.SIPpath)
logging.debug('self.AIPpath: %s', self.AIPpath)
if self.metatype in [1,2,3]:
if self.ok:
###########################################################
# get object_list from PREMIS file
self.Premis_filepath = '%s/%s/%s_PREMIS.xml' % (self.SIPpath,self.ObjectIdentifierValue,self.ObjectIdentifierValue)
self.object_list,errno,why = ESSMD.getPremisObjects(FILENAME=self.Premis_filepath)
# list [objectIdentifierValue,messageDigestAlgorithm,messageDigest,messageDigestOriginator,size,formatName,formatVersion]
if errno == 0:
logging.info('Succeeded to get object_list from premis for information package: %s', self.ObjectIdentifierValue)
else:
self.event_info = 'Problem to get object_list from premis for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# create AIP content METS file
#self.firstPremisObjectFlag = 1
METS_agent_list = []
if self.metatype == 1:
############################################
# Object have metatype 1 (METS)
self.METS_LABEL = 'ESSArch AIP'
self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
# Get SIP Content METS information
self.METSfilepath = os.path.join(self.SIPpath,self.ObjectIdentifierValue + '/metadata/SIP/' + self.ObjectIdentifierValue + '_Content_METS.xml')
res_info, res_files, res_struct, error, why = ESSMD.getMETSFileList(FILENAME=self.METSfilepath)
for agent in res_info[2]:
if not (agent[0] == 'CREATOR' and agent[3] == 'SOFTWARE'):
METS_agent_list.append(agent)
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
elif self.metatype == 2:
############################################
# Object have metatype 2 (RES)
self.METS_LABEL = 'Imaging AIP RA'
self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
elif self.metatype == 3:
############################################
# Object have metatype 3 (ADDML)
self.METS_LABEL = 'Born Digital AIP RA'
self.tmp_object_id = ('%s/%s_ADDML.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'ADDML'])
self.tmp_object_id = ('%s/%s_PREMIS.xml') % (self.ObjectIdentifierValue,self.ObjectIdentifierValue)
self.tmp_object_size = os.stat(os.path.join(self.SIPpath,self.tmp_object_id))[6]
self.object_list.append([self.tmp_object_id,'', '', '', self.tmp_object_size, 'ARCHMETAxmlWrap', 'PREMIS'])
METS_agent_list.append(['ARCHIVIST','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','ORGANIZATION','','Riksarkivet',[]])
METS_agent_list.append(['CREATOR','INDIVIDUAL','',AgentIdentifierValue,[]])
METS_agent_list.append(['CREATOR', 'OTHER', 'SOFTWARE', 'ESSArch', ['VERSION=%s' % ProcVersion]])
self.firstPremisObjectFlag = 1
self.DataObjectNumItems = 0
self.DataObjectSize = 0
self.MetaObjectSize = 0
self.MetaObjectIdentifier = 'None'
for self.object in self.object_list:
self.filepath = os.path.join(self.SIPpath, self.object[0])
self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
self.a_filepath = self.object[0]
if self.firstPremisObjectFlag:
if self.object[0] == self.ObjectIdentifierValue:
logging.info('First premis object match information package: %s', self.ObjectIdentifierValue)
if self.metatype == 1:
self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis'])
elif self.metatype == 2:
self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis','mix'])
elif self.metatype == 3:
self.METSdoc = ESSMD.createMets(self.ObjectIdentifierValue,self.METS_LABEL,METS_agent_list,['premis','addml','xhtml'])
self.firstPremisObjectFlag = 0
continue
else:
self.event_info = 'First premis object do not match information package: %s, premis_object: %s' % (self.ObjectIdentifierValue,self.object[0])
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
elif os.access(self.filepath_iso,os.R_OK):
self.file_statinfo = os.stat(self.filepath_iso)
if self.metatype == 2 and self.object[5] == 'ARCHMETA':
############################################
# Object have metatype 2 and RES file
#self.file_ID = string.replace(self.object[0],'/','%')
self.file_ID = self.object[0]
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'Content description'
self.file_MIMETYPE = 'text/csv'
self.file_MDTYPE = 'OTHER'
self.file_OTHERMDTYPE = 'RES'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
elif self.object[5] == 'ARCHMETAxmlWrap' and self.object[6] == 'PREMIS':
############################################
# Object is a PREMIS XML file
self.file_MDTYPE = 'PREMIS'
self.file_OTHERMDTYPE = ''
elif self.object[5] == 'ARCHMETAxmlWrap':
############################################
# Object is a OTHER XML file
self.file_MDTYPE = 'OTHER'
self.file_OTHERMDTYPE = self.object[6]
elif self.metatype == 1:
############################################
# Object have metatype 1, convert PREMIS formatName to MIME-type, datafile
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'Datafiles'
self.file_MIMETYPE = ESSPGM.Check().PREMISformat2MIMEtype(self.object[5])
self.file_USE = 'Datafile'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
if self.file_MIMETYPE == 'unknown':
self.event_info = 'Problem to idetify MIMETYPE from PREMIS for: %s' % self.filepath
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
elif self.metatype == 2 and self.object[0][-12:] == 'TIFFEdit.RES':
############################################
# Object have metatype 2 and RES file
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'RA Information'
self.file_MIMETYPE = 'text/csv'
self.file_USE = 'RA Information'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
elif self.metatype == 2:
############################################
# Object have metatype 2 and datafile is an tiff image
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'RA Datafiles'
self.file_MIMETYPE = 'image/tiff'
self.file_USE = 'RA Datafile'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
elif self.metatype == 3:
############################################
# Object have metatype 3, convert PREMIS formatName to MIME-type, datafile
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'Datafiles'
self.file_MIMETYPE = ESSPGM.Check().PREMISformat2MIMEtype(self.object[5])
self.file_USE = 'Datafile'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
if self.file_MIMETYPE == 'unknown':
self.event_info = 'Problem to idetify MIMETYPE from PREMIS for: %s' % self.filepath
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
else:
############################################
# Object is a datafile
self.file_ID = self.object[0]
self.file_SIZE = self.file_statinfo.st_size
self.file_LABEL = 'Datafiles'
self.file_MIMETYPE = 'xxxxx' # Maste fixas
self.file_USE = 'Datafile'
self.file_CHECKSUMTYPE = self.object[1]
self.file_CHECKSUM = self.object[2]
self.file_LOCTYPE = 'URL'
self.file_xlink_type = 'simple'
else:
self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# add files to METS file
if not (self.object[5] == 'ARCHMETA' or self.object[5] == 'ARCHMETAxmlWrap'):
self.DataObjectNumItems += 1
self.DataObjectSize += self.file_SIZE
self.fil_utc_mtime = datetime.datetime.utcfromtimestamp(self.file_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.fil_lociso_mtime = self.fil_utc_mtime.astimezone(self.tz).isoformat()
self.METSdoc = ESSMD.AddDataFiles(self.METSdoc,self.file_LABEL,'FILES','',[(self.file_ID,self.file_SIZE,self.fil_lociso_mtime,self.file_MIMETYPE,'',self.file_USE,self.file_CHECKSUMTYPE,self.file_CHECKSUM,self.file_LOCTYPE,self.file_xlink_type)])
elif self.object[5] == 'ARCHMETA':
self.MetaObjectSize += self.file_SIZE
self.fil_utc_mtime = datetime.datetime.utcfromtimestamp(self.file_statinfo.st_mtime).replace(tzinfo=pytz.utc)
self.fil_lociso_mtime = self.fil_utc_mtime.astimezone(self.tz).isoformat()
self.METSdoc = ESSMD.AddContentFiles(self.METSdoc,self.file_LABEL,'',[(self.file_ID,self.file_SIZE,self.fil_lociso_mtime,self.file_MIMETYPE,self.file_MDTYPE,self.file_OTHERMDTYPE,self.file_CHECKSUMTYPE,self.file_CHECKSUM,self.file_LOCTYPE,self.file_xlink_type)])
elif self.object[5] == 'ARCHMETAxmlWrap':
logging.info('Wrap XML file: ' + self.a_filepath + ' to METS file')
self.file_xml,errno,why = ESSMD.parseFromFile(self.filepath)
if errno:
self.event_info = 'Failed to parse XML file: ' + str(self.filepath) + ' error: ' + str(why)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
self.METSdoc = ESSMD.AddContentEtree(self.METSdoc,[(self.file_xml,self.file_MDTYPE,self.file_OTHERMDTYPE)])
if self.ok:
########################
# Update root schemalocation and remove all other schemalocation
self.METSdoc,errno,why = ESSMD.updateSchemaLocation(self.METSdoc)
########################
# Update all ADMID in DOC
res,errno,why = ESSMD.updateFilesADMID(self.METSdoc)
########################
# Set xml_METS to self.METSdoc
xml_METS = self.METSdoc
if self.metatype in [4]:
dt = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
loc_dt_isoformat = dt.astimezone(self.tz).isoformat()
xml_METS = ESSMD.updatePackage(FILENAME=self.SIPmets_objpath,TYPE='AIP',CREATED=loc_dt_isoformat,metsDocumentID=self.Cmets_obj)
if self.ok:
########################
# Write METS file
errno,why = ESSMD.writeToFile(xml_METS,self.Cmets_objpath)
if errno:
self.event_info = 'Problem to write METS to file for AIP package: ' + str(self.p_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
time.sleep(2)
self.ok = 0
self.MetaObjectIdentifier = self.Cmets_obj
self.MetaObjectSize = 0
if self.ok:
###########################################################
# get object_list from METS
self.object_list,errno,why = ESSMD.getAIPObjects(FILENAME=self.Cmets_objpath)
if errno == 0:
logging.info('Succeeded to get object_list from METS for information package: %s', self.ObjectIdentifierValue)
else:
self.event_info = 'Problem to get object_list from METS for information package: %s, errno: %s, detail: %s' % (self.ObjectIdentifierValue,str(errno),str(why))
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# Insert METS file as first object in AIP package
self.tmp_object_size = os.stat(self.Cmets_objpath)[6]
self.object_list.insert(0,[self.Cmets_obj,'','',self.tmp_object_size,''])
if self.ok:
###########################################################
# create AIP package file
try:
errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 30, 5)
if errno: logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
logging.info('Create AIP Package: ' + self.p_objpath)
self.tarfile = tarfile.open(self.p_objpath, "w",)
except tarfile.TarError:
self.event_info = 'Problem to create AIP Package: ' + str(self.p_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# add files to AIP package file
self.startTarTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
self.firstPremisObjectFlag = 1
self.ObjectNumItems = 0
self.ObjectSize = 0
for self.object in self.object_list:
if self.metatype in [1,2,3]:
self.a_filepath = self.object[0]
elif self.metatype in [4]:
self.a_filepath = '%s/%s' % (self.ObjectIdentifierValue,self.object[0])
self.a_filepath_iso = ESSPGM.Check().unicode2str(self.a_filepath)
self.object_size = int(self.object[3])
if self.a_filepath == self.Cmets_obj or self.a_filepath == '%s/%s' % (self.ObjectIdentifierValue,self.Cmets_obj):
self.filepath = self.Cmets_objpath
if self.metatype in [4]:
self.MetaObjectSize = self.object_size
self.DataObjectSize = 0
self.DataObjectNumItems = 0
else:
self.filepath = os.path.join(self.SIPpath, self.a_filepath)
if self.metatype in [4]:
self.DataObjectSize += self.object_size
self.DataObjectNumItems += 1
self.filepath_iso = ESSPGM.Check().unicode2str(self.filepath)
if os.access(self.filepath_iso,os.R_OK):
if int(os.stat(self.filepath_iso)[6]) == self.object_size:
try:
self.ObjectNumItems += 1
self.tarinfo = self.tarfile.gettarinfo(self.filepath_iso, self.a_filepath_iso)
self.tarfile.addfile(self.tarinfo, file(self.filepath_iso))
logging.info('Add: ' + self.a_filepath + ' to AIP Package: ' + self.p_obj)
except tarfile.TarError:
self.event_info = 'Problem to add: ' + str(self.a_filepath) + ' to AIP Package: ' + str(self.p_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
else:
self.event_info = 'Filesize for object path: %s is %s and METS object size is %s. The sizes must match!' % (self.filepath,str(os.stat(self.filepath_iso)[6]),str(self.object_size))
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
else:
self.event_info = 'Object path: %s do not exist or is not readable!' % self.filepath
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# Close AIP package
try:
self.tarfile.close()
except tarfile.TarError:
self.event_info = 'Problem to close AIP package: ' + str(self.p_objpath)
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
self.ok = 0
if self.ok:
###########################################################
# Check if StatusActivity is OK
self.ObjectSize = os.stat(self.p_objpath)[6]
self.stopTarTime = datetime.timedelta(seconds=time.localtime()[5],minutes=time.localtime()[4],hours=time.localtime()[3])
self.TarTime = self.stopTarTime-self.startTarTime
self.WriteSize = int(self.ObjectSize)/1048576
if self.TarTime.seconds < 1: self.TarTime = datetime.timedelta(seconds=1) #Fix min time to 1 second if it is zero.
self.TarMBperSEC = int(self.WriteSize)/int(self.TarTime.seconds)
logging.info('Close AIP package: ' + self.p_obj)
logging.info('Succeeded to create AIP for: ' + self.ObjectIdentifierValue + ' , ' + str(self.TarMBperSEC) + ' MB/Sec and Time: ' + str(self.TarTime))
self.timestamp_utc = datetime.datetime.utcnow().replace(microsecond=0,tzinfo=pytz.utc)
self.timestamp_dst = self.timestamp_utc.astimezone(self.tz)
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ObjectPackageName',self.p_obj,
'ObjectSize',self.ObjectSize,
'ObjectNumItems',self.ObjectNumItems,
'ObjectMessageDigest','',
'ObjectPath','',
'MetaObjectIdentifier',self.MetaObjectIdentifier,
'MetaObjectSize',self.MetaObjectSize,
'DataObjectSize',self.DataObjectSize,
'DataObjectNumItems',self.DataObjectNumItems,
'CreateDate',self.timestamp_utc.replace(tzinfo=None),
'CreateAgentIdentifierValue',AgentIdentifierValue,
'StatusProcess','39',
'StatusActivity','0',
'LastEventDate',self.timestamp_utc.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue,
'LocalDBdatetime',self.timestamp_utc.replace(tzinfo=None)),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else: ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'0','',2,self.ObjectIdentifierValue)
if errno == 0 and self.ext_IngestTable:
ext_res,ext_errno,ext_why = ESSMSSQL.DB().action(self.ext_IngestTable,'UPD',('ObjectPackageName',self.p_obj,
'ObjectSize',self.ObjectSize,
'ObjectNumItems',self.ObjectNumItems,
'ObjectMessageDigest','',
'ObjectPath','',
'MetaObjectIdentifier',self.MetaObjectIdentifier,
'MetaObjectSize',self.MetaObjectSize,
'DataObjectSize',self.DataObjectSize,
'DataObjectNumItems',self.DataObjectNumItems,
'CreateDate',self.timestamp_dst.replace(tzinfo=None),
'CreateAgentIdentifierValue',AgentIdentifierValue,
'StatusProcess','39',
'StatusActivity','0',
'LastEventDate',self.timestamp_dst.replace(tzinfo=None),
'linkingAgentIdentifierValue',AgentIdentifierValue),
('ObjectIdentifierValue',self.ObjectIdentifierValue))
if ext_errno: logging.error('Failed to update External DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(ext_why))
else:
res,errno,why = ESSDB.DB().action(self.IngestTable,'UPD',('ExtDBdatetime',self.timestamp_utc.replace(tzinfo=None)),('ObjectIdentifierValue',self.ObjectIdentifierValue))
if errno: logging.error('Failed to update Local DB: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else:
errno,why = ESSPGM.DB().SetAIPstatus(self.IngestTable, self.ext_IngestTable, AgentIdentifierValue, self.ObjectUUID, 31, 4)
if errno:
logging.error('Failed to update DB status for AIP: ' + str(self.ObjectIdentifierValue) + ' error: ' + str(why))
else:
self.event_info = 'Failed to create AIP package: ' + self.p_obj
logging.error(self.event_info)
ESSPGM.Events().create('1030','','ESSArch AIPCreator',ProcVersion,'1',self.event_info,2,self.ObjectIdentifierValue)
db.close_old_connections()
self.mLock.release()
time.sleep(int(self.Time))
self.mDieFlag=0