def build_infile():
from DIRAC.Resources.Catalog.PoolXMLCatalog import PoolXMLCatalog
pm = PoolXMLCatalog('pool_xml_catalog.xml')
for Lfn in pm.getLfnsList():
pfn = pm.getPfnsByLfn(Lfn)['Replicas']['Uknown']
return pfn
def build_infile(runlist):
from DIRAC.Resources.Catalog.PoolXMLCatalog import PoolXMLCatalog
pm = PoolXMLCatalog('pool_xml_catalog.xml')
runlist = runlist + '.list'
f = open(runlist, 'a')
for Lfn in pm.getLfnsList():
pfn = pm.getPfnsByLfn(Lfn)['Replicas']['Uknown']
RunNum = pfn.split('dst_CTA_')[1].split('.root')[0]
pfn = RunNum + ' ' + '-1 ' + pfn
f.write(pfn)
f.write('\n')
f.close()
return DIRAC.S_OK()
def getOutputType(outputs, inputs, directory=''):
""" This function searches the directory for POOL XML catalog files and extracts the type of the pfn.
If not found, inherits from the type of the inputs
"""
if not isinstance(outputs, list):
outputs = [outputs]
catalog = PoolXMLCatalog(_getPoolCatalogs(directory))
# inputs - by lfn
generatedIn = False
typeFileIn = []
for fname in inputs:
try:
tFileIn = str(
catalog.getTypeByPfn(
str(catalog.getPfnsByLfn(fname)['Replicas'].values()[0])))
except KeyError:
tFileIn = None
if not tFileIn:
generatedIn = True
else:
typeFileIn.append(tFileIn)
if generatedIn and inputs:
raise ValueError('Could not find Type for inputs')
# outputs - by pfn
pfnTypesOut = {}
for fname in outputs:
tFileOut = str(catalog.getTypeByPfn(fname))
if not tFileOut:
if typeFileIn:
tFileOut = typeFileIn[0]
else:
tFileOut = 'ROOT'
pfnTypesOut[fname] = tFileOut
return pfnTypesOut
def execute(self,
production_id=None,
prod_job_id=None,
wms_job_id=None,
workflowStatus=None,
stepStatus=None,
wf_commons=None,
step_commons=None,
step_number=None,
step_id=None):
""" Main execution function.
"""
try:
super(MergeMDF,
self).execute(self.version, production_id, prod_job_id,
wms_job_id, workflowStatus, stepStatus,
wf_commons, step_commons, step_number, step_id)
poolCat = PoolXMLCatalog(self.poolXMLCatName)
self._resolveInputVariables()
stepOutputs, stepOutputTypes, _histogram = self._determineOutputs()
logLines = [
'#' * len(self.version), self.version, '#' * len(self.version)
]
localInputs = [
str(poolCat.getPfnsByLfn(x)['Replicas'].values()[0])
for x in self.stepInputData
]
inputs = ' '.join(localInputs)
cmd = 'cat %s > %s' % (inputs, self.outputFilePrefix + '.' +
stepOutputTypes[0])
logLines.append('\nExecuting merge operation...')
self.log.info('Executing "%s"' % cmd)
result = systemCall(timeout=600, cmdSeq=shlex.split(cmd))
if not result['OK']:
self.log.error(result)
logLines.append('Merge operation failed with result:\n%s' %
result)
return S_ERROR('Problem Executing Application')
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
self.log.info(stdout)
if stderr:
self.log.error(stderr)
if status:
msg = 'Non-zero status %s while executing "%s"' % (status, cmd)
self.log.info(msg)
logLines.append(msg)
return S_ERROR('Problem Executing Application')
self.log.info("Going to manage %s output" % self.applicationName)
self._manageAppOutput(stepOutputs)
# Still have to set the application status e.g. user job case.
self.setApplicationStatus(
'%s %s Successful' %
(self.applicationName, self.applicationVersion))
# Write to log file
msg = 'Produced merged MDF file'
self.log.info(msg)
logLines.append(msg)
logLines = [str(i) for i in logLines]
logLines.append('#EOF')
fopen = open(self.applicationLog, 'w')
fopen.write('\n'.join(logLines) + '\n')
fopen.close()
return S_OK('%s %s Successful' %
(self.applicationName, self.applicationVersion))
except Exception as e: #pylint:disable=broad-except
self.log.exception("Failure in MergeMDF execute module",
lException=e)
return S_ERROR(str(e))
finally:
super(MergeMDF, self).finalize(self.version)
