def updateFileStatus(self):
gLogger.info("updateFileStatus: Updating Status of task files")
#Get the transformations to be updated
status = self.am_getOption('UpdateFilesStatus',
['Active', 'Completing', 'Stopped'])
res = self._selectTransformations(transType=self.transType,
status=status,
agentType=[])
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
# Get the files which are in a UPDATE state
updateStatus = self.am_getOption(
'FileUpdateStatus',
['Submitted', 'Received', 'Waiting', 'Running'])
timeStamp = str(datetime.datetime.utcnow() -
datetime.timedelta(minutes=10))
condDict = {'TransformationID': transID, 'Status': ['Assigned']}
res = self.transClient.getTransformationFiles(
condDict=condDict, older=timeStamp, timeStamp='LastUpdate')
if not res['OK']:
gLogger.error(
"updateFileStatus: Failed to get transformation files to update.",
res['Message'])
continue
if not res['Value']:
gLogger.info(
"updateFileStatus: No files to be updated for transformation %s."
% transID)
continue
res = self.getSubmittedFileStatus(res['Value'])
if not res['OK']:
gLogger.error(
"updateFileStatus: Failed to get updated file statuses for transformation",
"%s %s" % (transID, res['Message']))
continue
statusDict = res['Value']
if not statusDict:
gLogger.info(
"updateFileStatus: No file statuses to be updated for transformation %s."
% transID)
continue
fileReport = FileReport(server=self.transClient.getServer())
for lfn, status in statusDict.items():
fileReport.setFileStatus(int(transID), lfn, status)
res = fileReport.commit()
if not res['OK']:
gLogger.error(
"updateFileStatus: Failed to update file status for transformation",
"%s %s" % (transID, res['Message']))
else:
for status, update in res['Value'].items():
gLogger.info(
"updateFileStatus: Updated %s files for %s to %s." %
(update, transID, status))
gLogger.info(
"updateFileStatus: Transformation file status update complete")
return S_OK()
def _getFileReporter(self):
"""just return the file reporter (object)"""
if "FileReport" in self.workflow_commons:
return self.workflow_commons["FileReport"]
fileReport = FileReport()
self.workflow_commons["FileReport"] = fileReport
return fileReport
def _getFileReporter(self):
""" just return the file reporter (object)
"""
if self.workflow_commons.has_key('FileReport'):
return self.workflow_commons['FileReport']
else:
fileReport = FileReport()
self.workflow_commons['FileReport'] = fileReport
return fileReport
def setFileStatus(self, production, lfn, status):
""" Set the file status for the given production in the Production Database
:param int production: production ID
:param string lfn: logical file name of the file that needs status change
:param string status: status to set
:return: S_OK(), S_ERROR()
"""
self.log.verbose('setFileStatus(%s,%s,%s)' %(production, lfn, status))
fileReport = self.workflow_commons.setdefault('FileReport', FileReport('Transformation/TransformationManager') )
result = fileReport.setFileStatus(production, lfn, status)
if not result['OK']:
self.log.warn(result['Message'])
self.workflow_commons['FileReport'] = fileReport
return result
def updateFileStatus( self, transIDOPBody, clients ):
""" Update the files status
"""
transID = transIDOPBody.keys()[0]
timeStamp = str( datetime.datetime.utcnow() - datetime.timedelta( minutes = 10 ) )
condDict = {'TransformationID' : transID, 'Status' : ['Assigned']}
transformationFiles = clients['TransformationClient'].getTransformationFiles( condDict = condDict,
older = timeStamp, timeStamp = 'LastUpdate' )
self._logDebug( "getTransformationFiles(%s) return value: %s" % ( str( condDict ), transformationFiles ),
method = 'updateFileStatus', transID = transID )
if not transformationFiles['OK']:
self._logError( "Failed to get transformation files to update: %s" % transformationFiles['Message'],
method = 'updateFileStatus' )
return transformationFiles
if not transformationFiles['Value']:
self._logInfo( "No files to be updated", transID = transID, method = 'updateFileStatus' )
return transformationFiles
submittedFileStatus = clients['TaskManager'].getSubmittedFileStatus( transformationFiles['Value'] )
self._logDebug( "getSubmittedFileStatus return value: %s" % submittedFileStatus,
method = 'updateFileStatus', transID = transID )
if not submittedFileStatus['OK']:
self._logError( "Failed to get updated file states for transformation: %s" % submittedFileStatus['Message'],
transID = transID, method = 'updateFileStatus' )
return submittedFileStatus
statusDict = submittedFileStatus['Value']
if not statusDict:
self._logInfo( "No file states to be updated", transID = transID, method = 'updateFileStatus' )
return submittedFileStatus
fileReport = FileReport( server = clients['TransformationClient'].getServer() )
for lfn, status in statusDict.items():
setFileStatus = fileReport.setFileStatus( transID, lfn, status )
if not setFileStatus['OK']:
return setFileStatus
commit = fileReport.commit()
if not commit['OK']:
self._logError( "Failed to update file states for transformation: %s" % commit['Message'],
transID = transID, method = 'updateFileStatus' )
return commit
else:
self._logInfo( "Updated the states of %d files" % len( commit['Value'] ),
transID = transID, method = 'updateFileStatus' )
return S_OK()
def updateFileStatus(self, transIDOPBody, clients):
""" Update the files status
"""
transID = transIDOPBody.keys()[0]
method = 'updateFileStatus'
timeStamp = str(datetime.datetime.utcnow() -
datetime.timedelta(minutes=10))
# get transformation files
condDict = {'TransformationID': transID, 'Status': ['Assigned']}
transformationFiles = clients[
'TransformationClient'].getTransformationFiles(
condDict=condDict, older=timeStamp, timeStamp='LastUpdate')
if not transformationFiles['OK']:
self._logError("Failed to get transformation files to update:",
transformationFiles['Message'],
method=method,
transID=transID)
return transformationFiles
if not transformationFiles['Value']:
self._logInfo("No files to be updated",
method=method,
transID=transID)
return transformationFiles
# Get the status of the transformation files
# Sort the files by taskID
taskFiles = {}
for fileDict in transformationFiles['Value']:
taskFiles.setdefault(fileDict['TaskID'], []).append(fileDict)
chunkSize = 100
self._logVerbose("Getting file status for %d tasks (chunks of %d)" %
(len(taskFiles), chunkSize),
method=method,
transID=transID)
updated = {}
# Process 100 tasks at a time
for nb, taskIDs in enumerate(breakListIntoChunks(taskFiles,
chunkSize)):
fileChunk = []
for taskID in taskIDs:
fileChunk += taskFiles[taskID]
submittedFileStatus = clients[
'TaskManager'].getSubmittedFileStatus(fileChunk)
if not submittedFileStatus['OK']:
self._logError(
"Failed to get updated file states for transformation:",
submittedFileStatus['Message'],
method=method,
transID=transID)
return submittedFileStatus
statusDict = submittedFileStatus['Value']
if not statusDict:
self._logVerbose("%4d: No file states to be updated" % nb,
method=method,
transID=transID)
continue
# Set the status of files
fileReport = FileReport(
server=clients['TransformationClient'].getServer())
for lfn, status in statusDict.iteritems():
updated[status] = updated.setdefault(status, 0) + 1
setFileStatus = fileReport.setFileStatus(transID, lfn, status)
if not setFileStatus['OK']:
return setFileStatus
commit = fileReport.commit()
if not commit['OK']:
self._logError(
"Failed to update file states for transformation:",
commit['Message'],
method=method,
transID=transID)
return commit
else:
self._logVerbose("%4d: Updated the states of %d files" %
(nb, len(commit['Value'])),
method=method,
transID=transID)
for status, nb in updated.iteritems():
self._logInfo("Updated %d files to status %s" % (nb, status),
method=method,
transID=transID)
return S_OK()
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error("Failed to resolve input parameters:", result['Message'])
return result
if not self.enable:
self.log.info('Module is disabled by control flag')
return S_OK('Module is disabled by control flag')
self.fileReport = self.fileReport if self.fileReport else FileReport('Transformation/TransformationManager')
if self.InputData:
inputFiles = self.fileReport.getFiles()
for lfn in self.InputData:
if lfn not in inputFiles:
self.log.verbose('No status populated for input data %s, setting to "Unused"' % lfn)
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.info('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
inputFiles = self.fileReport.getFiles()
for lfn in inputFiles:
if inputFiles[lfn] != 'ApplicationCrash' and self.jobType != "Split":
self.log.info('Forcing status to "Unused" due to workflow failure for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
else:
inputFiles = self.fileReport.getFiles()
if inputFiles:
self.log.info('Workflow status OK, setting input file status to Processed')
for lfn in inputFiles:
self.log.info('Setting status to "Processed" for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Processed')
fileReportCommitResult = self.fileReport.commit()
if fileReportCommitResult['OK']:
self.log.info('Status of files have been properly updated in the ProcessingDB')
else:
self.log.error('Failed to report file status to ProductionDB:', fileReportCommitResult['Message'])
self.log.error('Request will be generated.')
disetResult = self.fileReport.generateForwardDISET()
if not disetResult['OK']:
self.log.warn( "Could not generate Operation for file report with result:\n%s" % disetResult['Message'] )
else:
if disetResult['Value'] is None:
self.log.info( "Files correctly reported to TransformationDB" )
else:
request = self._getRequestContainer()
request.addOperation( disetResult['Value'] )
self.workflow_commons['Request'] = request
# Must ensure that the local job report instance is used to report the final status
# in case of failure and a subsequent failover operation
if self.workflowStatus['OK'] and self.stepStatus['OK']:
self.jobReport.setApplicationStatus('Job Finished Successfully')
resFF = self.generateFailoverFile()
if not resFF['OK']:
self.log.error(resFF['Message'])
return self.finalize()
def test__basicSuccess(self, _patch):
axlf = AnalyseXMLSummary(bkClient=bkc_mock, dm=dm_mock)
f = open('XMLSummaryFile', 'w')
f.write("""<?xml version="1.0" encoding="UTF-8"?>
<summary version="1.0" xsi:noNamespaceSchemaLocation="$XMLSUMMARYBASEROOT/xml/XMLSummary.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<success>True</success>
<step>finalize</step>
<usage>
<stat unit="KB" useOf="MemoryMaximum">866104.0</stat>
</usage>
<input>
<file GUID="CCE96707-4BE9-E011-81CD-003048F35252" name="LFN:00012478_00000532_1.sim" status="full">200</file>
</input>
<output>
<file GUID="229BBEF1-66E9-E011-BBD0-003048F35252" name="PFN:00012478_00000532_2.xdigi" status="full">200</file>
</output>
</summary>
""")
f.close()
axlf.XMLSummary_o = XMLSummary('XMLSummaryFile')
res = axlf._basicSuccess()
self.assertFalse(res)
axlf.XMLSummary_o.inputFileStats = {
'full': 2,
'part': 1,
'fail': 0,
'other': 0
}
axlf.XMLSummary_o.inputStatus = [('aa/1.txt', 'full'),
('aa/2.txt', 'part')]
axlf.inputDataList = ['aa/1.txt', 'aa/2.txt']
axlf.numberOfEvents = -1
axlf.fileReport = FileReport()
axlf.production_id = '123'
res = axlf._basicSuccess()
self.assertTrue(res)
self.assertEqual(axlf.fileReport.statusDict,
{'aa/2.txt': 'Problematic'})
axlf.XMLSummary_o.inputFileStats = {
'full': 2,
'part': 0,
'fail': 1,
'other': 0
}
axlf.XMLSummary_o.inputStatus = [('aa/1.txt', 'fail'),
('aa/2.txt', 'full')]
axlf.inputDataList = ['aa/1.txt', 'aa/2.txt']
axlf.numberOfEvents = -1
axlf.fileReport = FileReport()
axlf.production_id = '123'
res = axlf._basicSuccess()
self.assertTrue(res)
self.assertEqual(axlf.fileReport.statusDict,
{'aa/1.txt': 'Problematic'})
axlf.XMLSummary_o.inputFileStats = {
'full': 2,
'part': 0,
'fail': 1,
'other': 0
}
axlf.XMLSummary_o.inputStatus = [('aa/1.txt', 'fail'),
('aa/2.txt', 'full')]
axlf.inputDataList = ['aa/3.txt']
axlf.numberOfEvents = -1
axlf.fileReport = FileReport()
axlf.production_id = '123'
res = axlf._basicSuccess()
self.assertFalse(res)
self.assertEqual(axlf.fileReport.statusDict, {})
axlf.XMLSummary_o.inputFileStats = {
'full': 2,
'part': 1,
'fail': 1,
'other': 0
}
axlf.XMLSummary_o.inputStatus = [('aa/1.txt', 'fail'),
('aa/2.txt', 'part')]
axlf.inputDataList = ['aa/1.txt', 'aa/2.txt']
axlf.numberOfEvents = -1
axlf.fileReport = FileReport()
axlf.production_id = '123'
res = axlf._basicSuccess()
self.assertTrue(res)
self.assertEqual(axlf.fileReport.statusDict, {
'aa/1.txt': 'Problematic',
'aa/2.txt': 'Problematic'
})
axlf.XMLSummary_o.inputFileStats = {
'full': 2,
'part': 1,
'fail': 1,
'other': 0
}
axlf.XMLSummary_o.inputStatus = [('aa/1.txt', 'fail'),
('aa/2.txt', 'part')]
axlf.inputDataList = ['aa/1.txt', 'aa/2.txt']
axlf.numberOfEvents = '10'
axlf.fileReport = FileReport()
axlf.production_id = '123'
res = axlf._basicSuccess()
self.assertTrue(res)
self.assertEqual(axlf.fileReport.statusDict,
{'aa/1.txt': 'Problematic'})
