def execute(self): """ Execute it. """ jobDescription = self.paramDict['Job'] prodID = self.paramDict['TransformationID'] jobID = self.paramDict['TaskID'] inputData = self.paramDict['InputData'] job = Job(jobDescription) commons = job._getParameters() #pylint: disable=protected-access code = job.workflow.createCode() outputList = [] for line in code.split("\n"): if line.count("listoutput"): outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used commons['outputList'] = outputList commons['PRODUCTION_ID'] = prodID commons['JOB_ID'] = jobID if inputData: commons['InputData'] = inputData result = constructProductionLFNs(commons) if not result['OK']: LOG.error(result['Message']) return result if commons['JobType'] in ILDJOBTYPES and commons['InputData']: for index, outputFile in enumerate( result['Value']['ProductionOutputData'] ): outputFileILD = getProdFilenameFromInput( commons['InputData'], outputFile, prodID, jobID ) result['Value']['ProductionOutputData'][index] = outputFileILD LOG.debug("Changed output file name from '%s' to '%s' " % (outputFile, outputFileILD)) return result
def execute(self): """ Execute it. """ jobDescription = self.paramDict['Job'] prodID = self.paramDict['TransformationID'] jobID = self.paramDict['TaskID'] inputData = self.paramDict['InputData'] job = Job(jobDescription) commons = job._getParameters() code = job.workflow.createCode() outputList = [] for line in code.split("\n"): if line.count("listoutput"): outputList += eval(line.split("#")[0].split("=")[-1]) commons['outputList'] = outputList commons['PRODUCTION_ID'] = prodID commons['JOB_ID'] = jobID if inputData: commons['InputData'] = inputData gLogger.debug(commons) result = constructProductionLFNs(commons) if not result['OK']: gLogger.error(result['Message']) return result
def test_contructProductionLFNsim(self): """test ProductionOutputData construct sim LFN..................................................""" commons = {} commons['PRODUCTION_ID'] = 12345 commons['JOB_ID'] = 1234 commons['outputList'] = [{'outputFile':"something_sim.slcio", 'outputPath':'/ilc/prod/clic/test/SIM'}] result = constructProductionLFNs(commons) self.assertEqual( result['OK'], True)
def test_contructProductionLFNsim(self): """test ProductionOutputData construct sim LFN..................................................""" commons = {} commons['PRODUCTION_ID'] = 12345 commons['JOB_ID'] = 1234 commons['outputList'] = [{ 'outputFile': "something_sim.slcio", 'outputPath': '/ilc/prod/clic/test/SIM' }] result = constructProductionLFNs(commons) self.assertEqual(result['OK'], True)
def test_contructProductionLFNoutput(self): """test ProductionOutputData construct out LFN..................................................""" commons = {} commons['PRODUCTION_ID'] = 12345 commons['JOB_ID'] = 1234 commons['outputList'] = [{'outputFile':"something_gen.stdhep", 'outputPath':'/ilc/prod/clic/test/gen'}] result = constructProductionLFNs(commons) res = {'ProductionOutputData' : ["/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep"], 'LogFilePath' : ["/ilc/prod/clic/test/gen/00012345/LOG/001"], 'LogTargetPath' : ["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"]} for key in res.keys(): self.assertEqual( result['Value'][key], res[key])
def test_contructProductionLFNoutput(self): """test ProductionOutputData construct out LFN..................................................""" commons = {} commons['PRODUCTION_ID'] = 12345 commons['JOB_ID'] = 1234 commons['outputList'] = [{ 'outputFile': "something_gen.stdhep", 'outputPath': '/ilc/prod/clic/test/gen' }] result = constructProductionLFNs(commons) res = { 'ProductionOutputData': [ "/ilc/prod/clic/test/gen/00012345/001/something_gen_12345_1234.stdhep" ], 'LogFilePath': ["/ilc/prod/clic/test/gen/00012345/LOG/001"], 'LogTargetPath': ["/ilc/prod/clic/test/gen/LOG/00012345/00012345_1234.tar"] } for key in res.keys(): self.assertEqual(result['Value'][key], res[key])
def execute(self): """ Execute it. """ jobDescription = self.paramDict['Job'] prodID = self.paramDict['TransformationID'] jobID = self.paramDict['TaskID'] inputData = self.paramDict['InputData'] job = Job(jobDescription) commons = job._getParameters() #pylint: disable=protected-access code = job.workflow.createCode() outputList = [] for line in code.split("\n"): if line.count("listoutput"): outputList += eval(line.split("#")[0].split("=")[-1]) #pylint: disable=eval-used commons['outputList'] = outputList commons['PRODUCTION_ID'] = prodID commons['JOB_ID'] = jobID if inputData: commons['InputData'] = inputData result = constructProductionLFNs(commons) if not result['OK']: gLogger.error(result['Message']) return result if commons['JobType'] in ILDJOBTYPES and commons['InputData']: for index, outputFile in enumerate( result['Value']['ProductionOutputData']): outputFileILD = getProdFilenameFromInput( commons['InputData'], outputFile, prodID, jobID) result['Value']['ProductionOutputData'][index] = outputFileILD gLogger.debug("Changed output file name from '%s' to '%s' " % (outputFile, outputFileILD)) return result
def test_contructProductionLFNBad(self): """test ProductionOutputData construct Bad LFN..................................................""" commons = {} result = constructProductionLFNs(commons) self.assertEqual( result['OK'], False)
def test_contructProductionLFNBad(self): """test ProductionOutputData construct Bad LFN..................................................""" commons = {} result = constructProductionLFNs(commons) self.assertEqual(result['OK'], False)
def checkDescendents(self, transformation, filedict, jobFileDict): """ look that all jobs produced, or not output """ res = self.prodDB.getTransformationParameters(transformation, ['Body']) if not res['OK']: self.log.error('Could not get Body from TransformationDB') return res body = res['Value'] workflow = fromXMLString(body) workflow.resolveGlobalVars() olist = [] jtype = workflow.findParameter('JobType') if not jtype: self.log.error('Type for transformation %d was not defined' % transformation) return S_ERROR('Type for transformation %d was not defined' % transformation) for step in workflow.step_instances: param = step.findParameter('listoutput') if not param: continue olist.extend(param.value) expectedlfns = [] contactfailed = [] fileprocessed = [] files = [] tasks_to_be_checked = {} for files in jobFileDict.values(): for f in files: if f in filedict: tasks_to_be_checked[f] = filedict[f] #get the tasks that need to be checked for filep, task in tasks_to_be_checked.items(): commons = {} commons['outputList'] = olist commons['PRODUCTION_ID'] = transformation commons['JOB_ID'] = task commons['JobType'] = jtype out = constructProductionLFNs(commons) expectedlfns = out['Value']['ProductionOutputData'] res = self.replicaManager.getCatalogFileMetadata(expectedlfns) if not res['OK']: self.log.error('Getting metadata failed') contactfailed.append(filep) continue if not filep in files: files.append(filep) success = res['Value']['Successful'].keys() failed = res['Value']['Failed'].keys() if len(success) and not len(failed): fileprocessed.append(filep) final_list_unused = files for file_all in files: if file_all in fileprocessed: try: final_list_unused.remove(filep) except: self.log.warn("Item not in list anymore") result = {'filesprocessed' : fileprocessed, 'filesToMarkUnused' : final_list_unused} return S_OK(result)