def execute(self):
""" Execute it.
"""
jobDescription = self.paramDict['Job']
prodID = self.paramDict['TransformationID']
jobID = self.paramDict['TaskID']
inputData = self.paramDict['InputData']
job = Job(jobDescription)
commons = job._getParameters() #pylint: disable=protected-access
code = job.workflow.createCode()
outputList = []
for line in code.split("\n"):
if line.count("listoutput"):
outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used
commons['outputList'] = outputList
commons['PRODUCTION_ID'] = prodID
commons['JOB_ID'] = jobID
if inputData:
commons['InputData'] = inputData
result = constructProductionLFNs(commons)
if not result['OK']:
LOG.error(result['Message'])
return result
if commons['JobType'] in ILDJOBTYPES and commons['InputData']:
for index, outputFile in enumerate( result['Value']['ProductionOutputData'] ):
outputFileILD = getProdFilenameFromInput( commons['InputData'], outputFile, prodID, jobID )
result['Value']['ProductionOutputData'][index] = outputFileILD
LOG.debug("Changed output file name from '%s' to '%s' " % (outputFile, outputFileILD))
return result
def execute(self):
""" Execute it.
"""
jobDescription = self.paramDict['Job']
prodID = self.paramDict['TransformationID']
jobID = self.paramDict['TaskID']
inputData = self.paramDict['InputData']
job = Job(jobDescription)
commons = job._getParameters()
code = job.workflow.createCode()
outputList = []
for line in code.split("\n"):
if line.count("listoutput"):
outputList += eval(line.split("#")[0].split("=")[-1])
commons['outputList'] = outputList
commons['PRODUCTION_ID'] = prodID
commons['JOB_ID'] = jobID
if inputData:
commons['InputData'] = inputData
gLogger.debug(commons)
result = constructProductionLFNs(commons)
if not result['OK']:
gLogger.error(result['Message'])
return result
def test_contructProductionLFNsim(self):
"""test ProductionOutputData construct sim LFN.................................................."""
commons = {}
commons['PRODUCTION_ID'] = 12345
commons['JOB_ID'] = 1234
commons['outputList'] = [{'outputFile':"something_sim.slcio",
'outputPath':'/ilc/prod/clic/test/SIM'}]
result = constructProductionLFNs(commons)
self.assertEqual( result['OK'], True)
def test_contructProductionLFNsim(self):
"""test ProductionOutputData construct sim LFN.................................................."""
commons = {}
commons['PRODUCTION_ID'] = 12345
commons['JOB_ID'] = 1234
commons['outputList'] = [{
'outputFile': "something_sim.slcio",
'outputPath': '/ilc/prod/clic/test/SIM'
}]
result = constructProductionLFNs(commons)
self.assertEqual(result['OK'], True)
def test_contructProductionLFNoutput(self):
"""test ProductionOutputData construct out LFN.................................................."""
commons = {}
commons['PRODUCTION_ID'] = 12345
commons['JOB_ID'] = 1234
commons['outputList'] = [{'outputFile':"something_gen.stdhep",
'outputPath':'/ilc/prod/clic/test/gen'}]
result = constructProductionLFNs(commons)
res = {'ProductionOutputData' : ["/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep"],
'LogFilePath' : ["/ilc/prod/clic/test/gen/00012345/LOG/001"],
'LogTargetPath' : ["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"]}
for key in res.keys():
self.assertEqual( result['Value'][key], res[key])
def test_contructProductionLFNoutput(self):
"""test ProductionOutputData construct out LFN.................................................."""
commons = {}
commons['PRODUCTION_ID'] = 12345
commons['JOB_ID'] = 1234
commons['outputList'] = [{
'outputFile': "something_gen.stdhep",
'outputPath': '/ilc/prod/clic/test/gen'
}]
result = constructProductionLFNs(commons)
res = {
'ProductionOutputData': [
"/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep"
],
'LogFilePath': ["/ilc/prod/clic/test/gen/00012345/LOG/001"],
'LogTargetPath':
["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"]
}
for key in res.keys():
self.assertEqual(result['Value'][key], res[key])
def execute(self):
""" Execute it.
"""
jobDescription = self.paramDict['Job']
prodID = self.paramDict['TransformationID']
jobID = self.paramDict['TaskID']
inputData = self.paramDict['InputData']
job = Job(jobDescription)
commons = job._getParameters() #pylint: disable=protected-access
code = job.workflow.createCode()
outputList = []
for line in code.split("\n"):
if line.count("listoutput"):
outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used
commons['outputList'] = outputList
commons['PRODUCTION_ID'] = prodID
commons['JOB_ID'] = jobID
if inputData:
commons['InputData'] = inputData
result = constructProductionLFNs(commons)
if not result['OK']:
gLogger.error(result['Message'])
return result
if commons['JobType'] in ILDJOBTYPES and commons['InputData']:
for index, outputFile in enumerate(
result['Value']['ProductionOutputData']):
outputFileILD = getProdFilenameFromInput(
commons['InputData'], outputFile, prodID, jobID)
result['Value']['ProductionOutputData'][index] = outputFileILD
gLogger.debug("Changed output file name from '%s' to '%s' " %
(outputFile, outputFileILD))
return result
def test_contructProductionLFNBad(self):
"""test ProductionOutputData construct Bad LFN.................................................."""
commons = {}
result = constructProductionLFNs(commons)
self.assertEqual( result['OK'], False)
def test_contructProductionLFNBad(self):
"""test ProductionOutputData construct Bad LFN.................................................."""
commons = {}
result = constructProductionLFNs(commons)
self.assertEqual(result['OK'], False)
def checkDescendents(self, transformation, filedict, jobFileDict):
""" look that all jobs produced, or not output
"""
res = self.prodDB.getTransformationParameters(transformation, ['Body'])
if not res['OK']:
self.log.error('Could not get Body from TransformationDB')
return res
body = res['Value']
workflow = fromXMLString(body)
workflow.resolveGlobalVars()
olist = []
jtype = workflow.findParameter('JobType')
if not jtype:
self.log.error('Type for transformation %d was not defined' % transformation)
return S_ERROR('Type for transformation %d was not defined' % transformation)
for step in workflow.step_instances:
param = step.findParameter('listoutput')
if not param:
continue
olist.extend(param.value)
expectedlfns = []
contactfailed = []
fileprocessed = []
files = []
tasks_to_be_checked = {}
for files in jobFileDict.values():
for f in files:
if f in filedict:
tasks_to_be_checked[f] = filedict[f] #get the tasks that need to be checked
for filep, task in tasks_to_be_checked.items():
commons = {}
commons['outputList'] = olist
commons['PRODUCTION_ID'] = transformation
commons['JOB_ID'] = task
commons['JobType'] = jtype
out = constructProductionLFNs(commons)
expectedlfns = out['Value']['ProductionOutputData']
res = self.replicaManager.getCatalogFileMetadata(expectedlfns)
if not res['OK']:
self.log.error('Getting metadata failed')
contactfailed.append(filep)
continue
if not filep in files:
files.append(filep)
success = res['Value']['Successful'].keys()
failed = res['Value']['Failed'].keys()
if len(success) and not len(failed):
fileprocessed.append(filep)
final_list_unused = files
for file_all in files:
if file_all in fileprocessed:
try:
final_list_unused.remove(filep)
except:
self.log.warn("Item not in list anymore")
result = {'filesprocessed' : fileprocessed, 'filesToMarkUnused' : final_list_unused}
return S_OK(result)