def _manageAppOutput(self, outputs):
""" Calls self._findOutputs to find what's produced, then creates the LFNs
outputs, as called here, is created starting from step_commons['listoutput'],
but enriched with at least the outputDataName.
example of outputs:
[{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST',
'outputDataName': '00012345_00012345_2.BHADRON.DST'},
{'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST',
'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}]
:params list outputs: list of dicts of step output files descriptions
"""
if not outputs:
self.log.warn('Step outputs are not defined (normal user jobs. Not normal in productions and SAM jobs)')
return
else:
finalOutputs, _bkFileTypes = self._findOutputs(outputs)
self.log.info('Final step outputs are: %s' % (finalOutputs))
self.step_commons['listoutput'] = finalOutputs
if 'outputList' in self.workflow_commons:
for outFile in finalOutputs:
if outFile not in self.workflow_commons['outputList']:
self.workflow_commons['outputList'].append(outFile)
else:
self.workflow_commons['outputList'] = finalOutputs
if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons:
self.log.info('Attempting to recreate the production output LFNs...')
result = constructProductionLFNs(self.workflow_commons, self.bkClient)
if not result['OK']:
raise IOError("Could not create production LFNs: %s" % result['Message'])
self.workflow_commons['BookkeepingLFNs'] = result['Value']['BookkeepingLFNs']
self.workflow_commons['LogFilePath'] = result['Value']['LogFilePath']
self.workflow_commons['ProductionOutputData'] = result['Value']['ProductionOutputData']
def _resolveInputVariables(self):
""" By convention the module parameters are resolved here.
"""
super(UploadOutputData, self)._resolveInputVariables()
if 'outputDataStep' in self.workflow_commons:
self.outputDataStep = [
str(ds)
for ds in self.workflow_commons['outputDataStep'].split(';')
]
if 'outputList' in self.workflow_commons:
self.outputList = self.workflow_commons['outputList']
if 'outputMode' in self.workflow_commons:
self.outputMode = self.workflow_commons['outputMode']
# Use LHCb utility for local running via jobexec
if 'ProductionOutputData' in self.workflow_commons:
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData']
if isinstance(self.prodOutputLFNs, basestring):
self.prodOutputLFNs = [
i.strip() for i in self.prodOutputLFNs.split(';')
] # pylint: disable=no-member
else:
self.log.info(
"ProductionOutputData parameter not found, creating on the fly"
)
result = constructProductionLFNs(self.workflow_commons,
self.bkClient)
if not result['OK']:
self.log.error("Could not create production LFNs",
result['Message'])
return result
self.prodOutputLFNs = result['Value']['ProductionOutputData']
def _finalizeWithErrors(self, subj):
""" Method that sends an email and uploads intermediate job outputs.
"""
# Have to check that the output list is defined in the workflow commons, this is
# done by the first BK report module that executes at the end of a step but in
# this case the current step 'listoutput' must be added.
if 'outputList' in self.workflow_commons:
for outputItem in self.step_commons['listoutput']:
if outputItem not in self.workflow_commons['outputList']:
self.workflow_commons['outputList'].append(outputItem)
else:
self.workflow_commons['outputList'] = self.step_commons['listoutput']
result = constructProductionLFNs(self.workflow_commons, self.bkClient)
if not result['OK']:
self.log.error('Could not create production LFNs with message "%s"' % (result['Message']))
raise Exception(result['Message'])
if 'DebugLFNs' not in result['Value']:
self.log.error('No debug LFNs found after creating production LFNs, result was:%s' % result)
raise Exception('DebugLFNs Not Found')
debugLFNs = result['Value']['DebugLFNs']
subject = '[' + self.siteName + '][' + self.applicationName + '] ' + self.applicationVersion + \
": " + subj + ' ' + self.production_id + '_' + self.prod_job_id + ' JobID=' + str(self.jobID)
msg = 'The Application ' + self.applicationName + ' ' + self.applicationVersion + ' had a problem \n'
msg = msg + 'at site ' + self.siteName + '\n'
msg = msg + 'JobID is ' + str(self.jobID) + '\n'
msg = msg + 'JobName is ' + self.production_id + '_' + self.prod_job_id + '\n'
toUpload = {}
for lfn in debugLFNs:
if os.path.exists(os.path.basename(lfn)):
toUpload[os.path.basename(lfn)] = lfn
if toUpload:
msg += '\n\nIntermediate job data files:\n'
for fname, lfn in toUpload.items():
guidResult = getGUID(fname)
guidInput = ''
if not guidResult['OK']:
self.log.error('Could not find GUID for %s with message' % (fname), guidResult['Message'])
elif guidResult['generated']:
self.log.info('PoolXMLFile generated GUID(s) for the following files ', ', '.join(guidResult['generated']))
guidInput = guidResult['Value'][fname]
else:
guidInput = guidResult['Value'][fname]
if self._WMSJob():
self.log.info('Attempting: dm.putAndRegister("%s","%s","%s","%s") on master catalog' % (fname, lfn, guidInput,
self.debugSE))
result = DataManager(masterCatalogOnly=True).putAndRegister(lfn, fname, self.debugSE, guidInput)
self.log.info(result)
if not result['OK']:
self.log.error('Could not save INPUT data file with result', str(result['Message']))
msg += 'Could not save intermediate data file %s with result\n%s\n' % (fname, result['Message'])
else:
msg = msg + lfn + '\n' + str(result) + '\n'
else:
self.log.info("JOBID is null, would have attempted to upload: LFN:%s, file %s, GUID %s to %s" % (lfn, fname,
guidInput,
self.debugSE))
if not self._WMSJob():
self.log.info("JOBID is null, *NOT* sending mail, for information the mail was:\n====>Start\n%s\n<====End"
% (msg))
else:
mailAddress = self.opsH.getValue('EMail/JobFailures', '*****@*****.**')
self.log.info('Sending crash mail for job to %s' % (mailAddress))
res = self.nc.sendMail(mailAddress, subject, msg, '*****@*****.**', localAttempt=False)
if not res['OK']:
self.log.warn("The mail could not be sent")
def test_constructProductionLFNs(self):
# + test with InputData
paramDict = {
'PRODUCTION_ID':
'12345',
'JOB_ID':
'54321',
'configVersion':
'test',
'configName':
'certification',
'JobType':
'MCSimulation',
'outputList': [{
'outputDataType': 'sim',
'outputDataSE': 'Tier1-RDST',
'outputDataName': '00012345_00054321_1.sim'
}, {
'outputDataType': 'digi',
'outputDataSE': 'Tier1-RDST',
'outputDataName': '00012345_00054321_2.digi'
}, {
'outputDataType': 'dst',
'outputDataSE': 'Tier1_MC_M-DST',
'outputDataName': '00012345_00054321_4.dst'
}, {
'outputDataType':
'ALLSTREAMS.DST',
'outputBKType':
'ALLSTREAMS.DST',
'outputDataSE':
'Tier1_MC_M-DST',
'outputDataName':
'00012345_00054321_5.AllStreams.dst'
}],
}
AllStreamsDST = '/lhcb/certification/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst'
reslist = [
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
]
},
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst'
]
},
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst'
]
},
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst'
]
},
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [AllStreamsDST]
},
{
'LogTargetPath': [
'/lhcb/certification/test/LOG/00012345/0005/00012345_00054321.tar'
],
'LogFilePath':
['/lhcb/certification/test/LOG/00012345/0005/00054321'],
'DebugLFNs': [
'/lhcb/debug/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/debug/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/debug/test/DST/00012345/0005/00012345_00054321_4.dst',
'/lhcb/debug/test/ALLSTREAMS.DST/00012345/0005/00012345_00054321_5.AllStreams.dst',
'/lhcb/debug/test/CORE/00012345/0005/00054321_core'
],
'BookkeepingLFNs': [
'/lhcb/certification/test/SIM/00012345/0005/00012345_00054321_1.sim',
'/lhcb/certification/test/DIGI/00012345/0005/00012345_00054321_2.digi',
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
],
'ProductionOutputData': [
'/lhcb/certification/test/DST/00012345/0005/00012345_00054321_4.dst',
AllStreamsDST
]
},
]
outputDataFileMasks = ('', 'dst', 'DST', ['digi', 'dst'],
'ALLSTREAMS.DST', ['dst', 'allstreams.dst'])
for outputDataFileMask, resL in itertools.izip(outputDataFileMasks,
reslist):
paramDict['outputDataFileMask'] = outputDataFileMask
res = constructProductionLFNs(paramDict, bkc_mock)
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], resL)
resWithBkk = constructProductionLFNs(paramDict,
bkc_mock,
quick=False)
self.assertTrue(resWithBkk['OK'])
self.assertEqual(resWithBkk['Value'], resL)
def _resolveInputVariables(self):
""" By convention the module parameters are resolved here.
"""
super(BookkeepingReport, self)._resolveInputVariables()
super(BookkeepingReport, self)._resolveInputStep()
self.stepOutputs, _sot, _hist = self._determineOutputs()
# # VARS FROM WORKFLOW_COMMONS ##
if 'outputList' in self.workflow_commons:
for outputItem in self.stepOutputs:
if outputItem not in self.workflow_commons['outputList']:
self.workflow_commons['outputList'].append(outputItem)
else:
self.workflow_commons['outputList'] = self.stepOutputs
if 'BookkeepingLFNs' in self.workflow_commons and \
'LogFilePath' in self.workflow_commons and \
'ProductionOutputData' in self.workflow_commons:
logFilePath = self.workflow_commons['LogFilePath']
bkLFNs = self.workflow_commons['BookkeepingLFNs']
if not isinstance(bkLFNs, list):
bkLFNs = [i.strip() for i in bkLFNs.split(';')]
else:
self.log.info(
'LogFilePath / BookkeepingLFNs parameters not found, creating on the fly'
)
result = constructProductionLFNs(self.workflow_commons,
self.bkClient)
if not result['OK']:
self.log.error('Could not create production LFNs',
result['Message'])
raise ValueError('Could not create production LFNs')
bkLFNs = result['Value']['BookkeepingLFNs']
logFilePath = result['Value']['LogFilePath'][0]
self.ldate = time.strftime("%Y-%m-%d", time.localtime(time.time()))
self.ltime = time.strftime("%H:%M", time.localtime(time.time()))
if 'StartTime' in self.step_commons:
startTime = self.step_commons['StartTime']
self.ldatestart = time.strftime("%Y-%m-%d",
time.localtime(startTime))
self.ltimestart = time.strftime("%H:%M", time.localtime(startTime))
try:
self.xf_o = self.step_commons['XMLSummary_o']
except KeyError:
self.log.warn(
'XML Summary object not found, will try to create it (again?)')
try:
xmlSummaryFile = self.step_commons['XMLSummary']
except KeyError:
self.log.warn(
'XML Summary file name not found, will try to guess it')
xmlSummaryFile = 'summary%s_%s_%s_%s.xml' % (
self.applicationName, self.production_id, self.prod_job_id,
self.step_number)
self.log.warn('Trying %s' % xmlSummaryFile)
if xmlSummaryFile not in os.listdir('.'):
self.log.warn(
'XML Summary file %s not found, will try to guess a second time'
% xmlSummaryFile)
xmlSummaryFile = 'summary%s_%s.xml' % (
self.applicationName, self.step_id)
self.log.warn('Trying %s' % xmlSummaryFile)
if xmlSummaryFile not in os.listdir('.'):
self.log.warn(
'XML Summary file %s not found, will try to guess a third and last time'
% xmlSummaryFile)
xmlSummaryFile = 'summary%s_%s.xml' % (
self.applicationName, self.step_number)
self.log.warn('Trying %s' % xmlSummaryFile)
try:
self.xf_o = XMLSummary(xmlSummaryFile)
except XMLSummaryError as e:
self.log.warn('No XML summary available', '%s' % repr(e))
self.xf_o = None
return bkLFNs, logFilePath