def testCheckLumiInformation(self): """ _testCheckLumiInformation_ Test the function that checks if all files have run lumi information """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForRunLumiInformation(stepName="cmsRun1") self.assertNotEqual(myReport.getExitCode(), 70452) # Remove the lumi information on purpose myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step="cmsRun1") for fRef in fRefs: fRef.runs = ConfigSection() myReport2.checkForRunLumiInformation(stepName="cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName="cmsRun1")) self.assertEqual(myReport2.getExitCode(), 70452) return
def testCheckLumiInformation(self): """ _testCheckLumiInformation_ Test the function that checks if all files have run lumi information """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForRunLumiInformation(stepName = "cmsRun1") self.assertNotEqual(myReport.getExitCode(), 60452) # Remove the lumi information on purpose myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.runs = ConfigSection() myReport2.checkForRunLumiInformation(stepName = "cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport2.getExitCode(), 60452) return
def testASONoNameChange(self): AsyncStageOut_t.FakeTransferWorker.setFailProbability(0) testJob = self.roundtripHelper(preserveLFN = True) stepReport = Report('cmsRun1') stepReport.unpersist(testJob['fwjr_path']) files = stepReport.getAllFileRefsFromStep(step = 'cmsRun1') for file in files: self.assertNotEqual( file.lfn.find('store/temp'), -1, "The lfn should still have store/temp: %s" % file.lfn)
def testGetAdlerChecksum(self): """ _testGetAdlerChecksum_ Test the function that sees if all files have an adler checksum. For some reason, our default XML report doesn't have checksums Therefore it should fail. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForAdlerChecksum(stepName="cmsRun1") self.assertFalse(myReport.stepSuccessful(stepName="cmsRun1")) self.assertEqual(myReport.getExitCode(), 60451) # Now see what happens if the adler32 is set to None myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step="cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': None} myReport2.checkForAdlerChecksum(stepName="cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName="cmsRun1")) self.assertEqual(myReport2.getExitCode(), 60451) myReport3 = Report("cmsRun1") myReport3.parse(self.xmlPath) fRefs = myReport3.getAllFileRefsFromStep(step="cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': 100} myReport3.checkForAdlerChecksum(stepName="cmsRun1") self.assertTrue(myReport3.getExitCode() != 60451) return
def testGetAdlerChecksum(self): """ _testGetAdlerChecksum_ Test the function that sees if all files have an adler checksum. For some reason, our default XML report doesn't have checksums Therefore it should fail. """ myReport = Report("cmsRun1") myReport.parse(self.xmlPath) myReport.checkForAdlerChecksum(stepName = "cmsRun1") self.assertFalse(myReport.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport.getExitCode(), 60451) # Now see what happens if the adler32 is set to None myReport2 = Report("cmsRun1") myReport2.parse(self.xmlPath) fRefs = myReport2.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': None} myReport2.checkForAdlerChecksum(stepName = "cmsRun1") self.assertFalse(myReport2.stepSuccessful(stepName = "cmsRun1")) self.assertEqual(myReport2.getExitCode(), 60451) myReport3 = Report("cmsRun1") myReport3.parse(self.xmlPath) fRefs = myReport3.getAllFileRefsFromStep(step = "cmsRun1") for fRef in fRefs: fRef.checksums = {'adler32': 100} myReport3.checkForAdlerChecksum(stepName = "cmsRun1") self.assertTrue(myReport3.getExitCode() != 60451) return
def post(self, emulator=None): """ _post_ Post execution checkpointing """ # Another emulator check if emulator is not None: return emulator.emulatePost(self.step) logging.info("Steps.Executors.%s.post called", self.__class__.__name__) for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: # Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s", step) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s", step, stepLocation) continue # First, get everything from a file and 'unpersist' it stepReport = Report(step) stepReport.unpersist(reportLocation) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue files = stepReport.getAllFileRefsFromStep(step=step) for fileInfo in files: if hasattr(fileInfo, 'lfn') and hasattr( fileInfo, 'location') and hasattr(fileInfo, 'guid'): fileInfo.user_dn = getattr(self.step, "userDN", None) fileInfo.async_dest = getattr(self.step, "asyncDest", None) fileInfo.user_vogroup = getattr(self.step, "owner_vogroup", '') fileInfo.user_vorole = getattr(self.step, "owner_vorole", '') stepReport.persist(reportLocation) return None
def post(self, emulator = None): """ _post_ Post execution checkpointing """ #Another emulator check if (emulator != None): return emulator.emulatePost( self.step ) for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: #Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s" % step) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s" \ % (step, stepLocation)) continue # First, get everything from a file and 'unpersist' it stepReport = Report(step) stepReport.unpersist(reportLocation) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue files = stepReport.getAllFileRefsFromStep(step = step) for file in files: if not hasattr(file, 'lfn') or not hasattr(file, 'location') or \ not hasattr(file, 'guid'): continue file.user_dn = getattr(self.step, "userDN", None) file.async_dest = getattr(self.step, "asyncDest", None) file.user_vogroup = getattr(self.step, "owner_vogroup", '') file.user_vorole = getattr(self.step, "owner_vorole", '') stepReport.persist(reportLocation) print "Steps.Executors.StageOut.post called" return None
def setJobWantsASO(self, filename, preserveLFN = True): stepReport = Report('cmsRun1') stepReport.unpersist(filename) files = stepReport.getAllFileRefsFromStep(step = 'cmsRun1') for file in files: if not hasattr(file, 'lfn') or not hasattr(file, 'location') or \ not hasattr(file, 'guid'): continue file.user_dn = "/CN=dummy-name/O=melopartydotcom" file.async_dest = "T2_US_Vanderbilt" file.user_vogroup = '' file.user_vorole = '' file.preserve_lfn = preserveLFN stepReport.persist(filename)
def execute(self, emulator = None): """ _execute_ """ #Are we using emulators again? if (emulator != None): return emulator.emulate( self.step, self.job ) overrides = {} if hasattr(self.step, 'override'): overrides = self.step.override.dictionary_() # Set wait to over an hour waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount)) logging.info("StageOut override is: %s " % self.step) # Pull out StageOutMgr Overrides # switch between old stageOut behavior and new, fancy stage out behavior useNewStageOutCode = False if overrides.has_key('newStageOut') and overrides.get('newStageOut'): useNewStageOutCode = True stageOutCall = {} if overrides.has_key("command") and overrides.has_key("option") \ and overrides.has_key("se-name") and overrides.has_key("lfn-prefix"): logging.critical('using override in StageOut') stageOutCall['command'] = overrides.get('command') stageOutCall['option'] = overrides.get('option') stageOutCall['se-name'] = overrides.get('se-name') stageOutCall['lfn-prefix'] = overrides.get('lfn-prefix') # naw man, this is real # iterate over all the incoming files if not useNewStageOutCode: # old style manager = StageOutMgr.StageOutMgr(**stageOutCall) manager.numberOfRetries = self.step.retryCount manager.retryPauseTime = self.step.retryDelay else: # new style logging.critical("STAGEOUT IS USING NEW STAGEOUT CODE") print "STAGEOUT IS USING NEW STAGEOUT CODE" manager = WMCore.Storage.FileManager.StageOutMgr( retryPauseTime = self.step.retryDelay, numberOfRetries = self.step.retryCount, **stageOutCall) # We need to find a list of steps in our task # And eventually a list of jobReports for out steps # Search through steps for report files filesTransferred = [] for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: #Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s" % (step)) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s" \ % (step, stepLocation)) continue # First, get everything from a file and 'unpersist' it stepReport = Report() stepReport.unpersist(reportLocation, step) taskID = getattr(stepReport.data, 'id', None) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue # Okay, time to start using stuff # Now I'm a bit confused about this; each report should ONLY # Have the results of that particular step in it, # So getting all the files should get ONLY the files # for that step; or so I hope files = stepReport.getAllFileRefsFromStep(step = step) for file in files: if not hasattr(file, 'lfn') and hasattr(file, 'pfn'): # Then we're truly hosed on this file; ignore it msg = "Not a file: %s" % file logging.error(msg) continue # Support direct-to-merge # This requires pulling a bunch of stuff from everywhere # First check if it's needed if hasattr(self.step.output, 'minMergeSize') \ and hasattr(file, 'size') \ and not getattr(file, 'merged', False): # We need both of those to continue, and we don't # direct-to-merge if getattr(self.step.output, 'doNotDirectMerge', False): # Then we've been told explicitly not to do direct-to-merge continue if file.size >= self.step.output.minMergeSize: # Then this goes direct to merge try: file = self.handleLFNForMerge(mergefile = file, step = step) except Exception, ex: logging.error("Encountered error while handling LFN for merge due to size.\n") logging.error(str(ex)) logging.debug(file) logging.debug("minMergeSize: %s" % self.step.output.minMergeSize) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60401, "DirectToMergeFailure", str(ex)) elif getattr(self.step.output, 'maxMergeEvents', None) != None\ and getattr(file, 'events', None) != None\ and not getattr(file, 'merged', False): # Then direct-to-merge due to events if # the file is large enough: if file.events >= self.step.output.maxMergeEvents: # straight to merge try: file = self.handleLFNForMerge(mergefile = file, step = step) except Exception, ex: logging.error("Encountered error while handling LFN for merge due to events.\n") logging.error(str(ex)) logging.debug(file) logging.debug("maxMergeEvents: %s" % self.step.output.maxMergeEvents) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60402, "DirectToMergeFailure", str(ex)) # Save the input PFN in case we need it # Undecided whether to move file.pfn to the output PFN file.InputPFN = file.pfn lfn = getattr(file, 'lfn') fileSource = getattr(file, 'Source', None) if fileSource in ['TFileService', 'UserDefined']: userLfnRegEx(lfn) else: lfnRegEx(lfn) fileForTransfer = {'LFN': lfn, 'PFN': getattr(file, 'pfn'), 'SEName' : None, 'StageOutCommand': None} signal.signal(signal.SIGALRM, alarmHandler) signal.alarm(waitTime) try: manager(fileForTransfer) #Afterwards, the file should have updated info. filesTransferred.append(fileForTransfer) file.StageOutCommand = fileForTransfer['StageOutCommand'] file.location = fileForTransfer['SEName'] file.OutputPFN = fileForTransfer['PFN'] except Alarm: msg = "Indefinite hang during stageOut of logArchive" logging.error(msg) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60403, "StageOutTimeout", msg) stepReport.persist("Report.pkl") except Exception, ex: manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60307, "StageOutFailure", str(ex)) stepReport.setStepStatus(self.stepName, 1) stepReport.persist("Report.pkl") raise
def testReportHandling(self): """ _testReportHandling_ Verify that we're able to parse a CMSSW report, convert it to a Report() style report, pickle it and then have the accountant process it. """ self.procPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(self.procPath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl") cmsRunStep = myReport.retrieveStep("cmsRun1") cmsRunStep.status = 0 myReport.setTaskName('/TestWF/None') myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath) pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000) pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") pFile.setDatasetPath("/bogus/dataset/path") #pFile.addRun(Run(1, *[45])) pFile.create() config = self.createConfig(workerThreads = 1) accountant = JobAccountantPoller(config) accountant.setup() accountant.algorithm() self.verifyJobSuccess(self.testJob["id"]) self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1")) inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root") inputFile.load() self.testMergeJob = Job(name = "testMergeJob", files = [inputFile]) self.testMergeJob.create(group = self.mergeJobGroup) self.testMergeJob["state"] = "complete" self.stateChangeAction.execute(jobs = [self.testMergeJob]) self.mergePath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml") myReport = Report("mergeReco") myReport.parse(self.mergePath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("mergeReco"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1", "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1", "dataTier": "RECO"} fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl") myReport.setTaskName('/MergeWF/None') cmsRunStep = myReport.retrieveStep("mergeReco") cmsRunStep.status = 0 myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath) accountant.algorithm() self.verifyJobSuccess(self.testMergeJob["id"]) self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco")) return
def testReportHandling(self): """ _testReportHandling_ Verify that we're able to parse a CMSSW report, convert it to a Report() style report, pickle it and then have the accountant process it. """ self.procPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(self.procPath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl") cmsRunStep = myReport.retrieveStep("cmsRun1") cmsRunStep.status = 0 myReport.setTaskName('/TestWF/None') myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath) pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000) pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") pFile.setDatasetPath("/bogus/dataset/path") #pFile.addRun(Run(1, *[45])) pFile.create() config = self.createConfig(workerThreads = 1) accountant = JobAccountantPoller(config) accountant.setup() accountant.algorithm() self.verifyJobSuccess(self.testJob["id"]) self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1")) inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root") inputFile.load() self.testMergeJob = Job(name = "testMergeJob", files = [inputFile]) self.testMergeJob.create(group = self.mergeJobGroup) self.testMergeJob["state"] = "complete" self.stateChangeAction.execute(jobs = [self.testMergeJob]) self.mergePath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml") myReport = Report("mergeReco") myReport.parse(self.mergePath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("mergeReco"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1", "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1", "dataTier": "RECO"} fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl") myReport.setTaskName('/MergeWF/None') cmsRunStep = myReport.retrieveStep("mergeReco") cmsRunStep.status = 0 myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath) accountant.algorithm() self.verifyJobSuccess(self.testMergeJob["id"]) self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco")) return
def execute(self, emulator = None): """ _execute_ """ #Are we using emulators again? if (emulator != None): return emulator.emulate( self.step, self.job ) overrides = {} if hasattr(self.step, 'override'): overrides = self.step.override.dictionary_() # Set wait to two hours per retry # this alarm leaves a subprocess behing that may cause trouble, see #6273 waitTime = overrides.get('waitTime', 7200 * self.step.retryCount) logging.info("StageOut override is: %s ", self.step) # Pull out StageOutMgr Overrides # switch between old stageOut behavior and new, fancy stage out behavior useNewStageOutCode = False if getattr(self.step, 'newStageout', False) or \ ('newStageOut' in overrides and overrides.get('newStageOut')): useNewStageOutCode = True stageOutCall = {} if "command" in overrides and "option" in overrides \ and "phedex-node" in overrides \ and"lfn-prefix" in overrides: logging.critical('using override in StageOut') stageOutCall['command'] = overrides.get('command') stageOutCall['option'] = overrides.get('option') stageOutCall['phedex-node']= overrides.get('phedex-node') stageOutCall['lfn-prefix'] = overrides.get('lfn-prefix') # naw man, this is real # iterate over all the incoming files if not useNewStageOutCode: # old style manager = StageOutMgr(**stageOutCall) manager.numberOfRetries = self.step.retryCount manager.retryPauseTime = self.step.retryDelay else: # new style logging.critical("STAGEOUT IS USING NEW STAGEOUT CODE") print("STAGEOUT IS USING NEW STAGEOUT CODE") manager = FMStageOutMgr(retryPauseTime = self.step.retryDelay, numberOfRetries = self.step.retryCount, **stageOutCall) # We need to find a list of steps in our task # And eventually a list of jobReports for out steps # Search through steps for report files filesTransferred = [] for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: #Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s", step) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s", step, stepLocation) continue # First, get everything from a file and 'unpersist' it stepReport = Report() stepReport.unpersist(reportLocation, step) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue # Okay, time to start using stuff # Now I'm a bit confused about this; each report should ONLY # Have the results of that particular step in it, # So getting all the files should get ONLY the files # for that step; or so I hope files = stepReport.getAllFileRefsFromStep(step = step) for fileName in files: # make sure the file information is consistent if hasattr(fileName, 'pfn') and ( not hasattr(fileName, 'lfn') or not hasattr(fileName, 'module_label') ): msg = "Not a valid file: %s" % fileName logging.error(msg) continue # Figuring out if we should do straight to merge # - should we do straight to merge at all ? # - is straight to merge disabled for this output ? # - are we over the size threshold # - are we over the event threshold ? straightToMerge = False if not getattr(fileName, 'merged', False) and hasattr(self.step.output, 'minMergeSize'): if fileName.module_label not in getattr(self.step.output, 'forceUnmergedOutputs', []): if getattr(fileName, 'size', 0) >= self.step.output.minMergeSize: straightToMerge = True if getattr(fileName, 'events', 0) >= getattr(self.step.output, 'maxMergeEvents', sys.maxsize): straightToMerge = True if straightToMerge: try: fileName = self.handleLFNForMerge(mergefile = fileName, step = step) except Exception as ex: logging.info("minMergeSize: %s", getattr(self.step.output, 'minMergeSize', None)) logging.info("maxMergeEvents: %s", getattr(self.step.output, 'maxMergeEvents', None)) logging.error("Encountered error while handling LFN for merge %s", fileName) logging.error(str(ex)) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60401, "DirectToMergeFailure", str(ex)) # Save the input PFN in case we need it # Undecided whether to move fileName.pfn to the output PFN fileName.InputPFN = fileName.pfn lfn = getattr(fileName, 'lfn') fileSource = getattr(fileName, 'Source', None) if fileSource in ['TFileService', 'UserDefined']: userLfnRegEx(lfn) else: lfnRegEx(lfn) fileForTransfer = {'LFN': lfn, 'PFN': getattr(fileName, 'pfn'), 'PNN' : None, 'StageOutCommand': None, 'Checksums' : getattr(fileName, 'checksums', None)} signal.signal(signal.SIGALRM, alarmHandler) signal.alarm(waitTime) try: manager(fileForTransfer) #Afterwards, the file should have updated info. filesTransferred.append(fileForTransfer) fileName.StageOutCommand = fileForTransfer['StageOutCommand'] fileName.location = fileForTransfer['PNN'] fileName.OutputPFN = fileForTransfer['PFN'] except Alarm: msg = "Indefinite hang during stageOut of logArchive" logging.error(msg) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60403, "StageOutTimeout", msg) stepReport.setStepStatus(self.stepName, 1) # well, if it fails for one file, it fails for the whole job... break except Exception as ex: manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60307, "StageOutFailure", str(ex)) stepReport.setStepStatus(self.stepName, 1) stepReport.persist(reportLocation) raise signal.alarm(0) # Am DONE with report. Persist it stepReport.persist(reportLocation) #Done with all steps, and should have a list of #stagedOut files in fileForTransfer logging.info("Transferred %i files", len(filesTransferred)) return
def execute(self, emulator=None): """ _execute_ """ #Are we using emulators again? if (emulator != None): return emulator.emulate(self.step, self.job) overrides = {} if hasattr(self.step, 'override'): overrides = self.step.override.dictionary_() # Set wait to two hours per retry # this alarm leaves a subprocess behing that may cause trouble, see #6273 waitTime = overrides.get('waitTime', 7200 * self.step.retryCount) logging.info("StageOut override is: %s ", self.step) # Pull out StageOutMgr Overrides # switch between old stageOut behavior and new, fancy stage out behavior useNewStageOutCode = False if getattr(self.step, 'newStageout', False) or \ ('newStageOut' in overrides and overrides.get('newStageOut')): useNewStageOutCode = True stageOutCall = {} if "command" in overrides and "option" in overrides \ and "phedex-node" in overrides \ and"lfn-prefix" in overrides: logging.critical('using override in StageOut') stageOutCall['command'] = overrides.get('command') stageOutCall['option'] = overrides.get('option') stageOutCall['phedex-node'] = overrides.get('phedex-node') stageOutCall['lfn-prefix'] = overrides.get('lfn-prefix') # naw man, this is real # iterate over all the incoming files if not useNewStageOutCode: # old style manager = StageOutMgr(**stageOutCall) manager.numberOfRetries = self.step.retryCount manager.retryPauseTime = self.step.retryDelay else: # new style logging.critical("STAGEOUT IS USING NEW STAGEOUT CODE") print("STAGEOUT IS USING NEW STAGEOUT CODE") manager = FMStageOutMgr(retryPauseTime=self.step.retryDelay, numberOfRetries=self.step.retryCount, **stageOutCall) # We need to find a list of steps in our task # And eventually a list of jobReports for out steps # Search through steps for report files filesTransferred = [] for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: #Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s", step) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s", step, stepLocation) continue # First, get everything from a file and 'unpersist' it stepReport = Report() stepReport.unpersist(reportLocation, step) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue # Okay, time to start using stuff # Now I'm a bit confused about this; each report should ONLY # Have the results of that particular step in it, # So getting all the files should get ONLY the files # for that step; or so I hope files = stepReport.getAllFileRefsFromStep(step=step) for fileName in files: # make sure the file information is consistent if hasattr(fileName, 'pfn') and (not hasattr(fileName, 'lfn') or not hasattr(fileName, 'module_label')): msg = "Not a valid file: %s" % fileName logging.error(msg) continue # Figuring out if we should do straight to merge # - should we do straight to merge at all ? # - is straight to merge disabled for this output ? # - are we over the size threshold # - are we over the event threshold ? straightToMerge = False if not getattr(fileName, 'merged', False) and hasattr( self.step.output, 'minMergeSize'): if fileName.module_label not in getattr( self.step.output, 'forceUnmergedOutputs', []): if getattr(fileName, 'size', 0) >= self.step.output.minMergeSize: straightToMerge = True if getattr(fileName, 'events', 0) >= getattr( self.step.output, 'maxMergeEvents', sys.maxsize): straightToMerge = True if straightToMerge: try: fileName = self.handleLFNForMerge(mergefile=fileName, step=step) except Exception as ex: logging.info( "minMergeSize: %s", getattr(self.step.output, 'minMergeSize', None)) logging.info( "maxMergeEvents: %s", getattr(self.step.output, 'maxMergeEvents', None)) logging.error( "Encountered error while handling LFN for merge %s", fileName) logging.error(str(ex)) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60401, "DirectToMergeFailure", str(ex)) # Save the input PFN in case we need it # Undecided whether to move fileName.pfn to the output PFN fileName.InputPFN = fileName.pfn lfn = getattr(fileName, 'lfn') fileSource = getattr(fileName, 'Source', None) if fileSource in ['TFileService', 'UserDefined']: userLfnRegEx(lfn) else: lfnRegEx(lfn) fileForTransfer = { 'LFN': lfn, 'PFN': getattr(fileName, 'pfn'), 'PNN': None, 'StageOutCommand': None, 'Checksums': getattr(fileName, 'checksums', None) } signal.signal(signal.SIGALRM, alarmHandler) signal.alarm(waitTime) try: manager(fileForTransfer) #Afterwards, the file should have updated info. filesTransferred.append(fileForTransfer) fileName.StageOutCommand = fileForTransfer[ 'StageOutCommand'] fileName.location = fileForTransfer['PNN'] fileName.OutputPFN = fileForTransfer['PFN'] except Alarm: msg = "Indefinite hang during stageOut of logArchive" logging.error(msg) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60403, "StageOutTimeout", msg) stepReport.setStepStatus(self.stepName, 1) # well, if it fails for one file, it fails for the whole job... break except Exception as ex: manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60307, "StageOutFailure", str(ex)) stepReport.setStepStatus(self.stepName, 1) stepReport.persist(reportLocation) raise signal.alarm(0) # Am DONE with report. Persist it stepReport.persist(reportLocation) #Done with all steps, and should have a list of #stagedOut files in fileForTransfer logging.info("Transferred %i files", len(filesTransferred)) return
def execute(self, emulator = None): """ _execute_ """ #Are we using emulators again? if (emulator != None): return emulator.emulate( self.step, self.job ) overrides = {} if hasattr(self.step, 'override'): overrides = self.step.override.dictionary_() # Set wait to over an hour waitTime = overrides.get('waitTime', 3600 + (self.step.retryDelay * self.step.retryCount)) logging.info("StageOut override is: %s " % self.step) # Pull out StageOutMgr Overrides # switch between old stageOut behavior and new, fancy stage out behavior useNewStageOutCode = False if getattr(self.step, 'newStageout', False) or \ ('newStageOut' in overrides and overrides.get('newStageOut')): useNewStageOutCode = True stageOutCall = {} if "command" in overrides and "option" in overrides \ and "se-name" in overrides and "phedex-node" in overrides \ and"lfn-prefix" in overrides: logging.critical('using override in StageOut') stageOutCall['command'] = overrides.get('command') stageOutCall['option'] = overrides.get('option') stageOutCall['se-name'] = overrides.get('se-name') stageOutCall['phedex-node']= overrides.get('phedex-node') stageOutCall['lfn-prefix'] = overrides.get('lfn-prefix') # naw man, this is real # iterate over all the incoming files if not useNewStageOutCode: # old style manager = StageOutMgr.StageOutMgr(**stageOutCall) manager.numberOfRetries = self.step.retryCount manager.retryPauseTime = self.step.retryDelay else: # new style logging.critical("STAGEOUT IS USING NEW STAGEOUT CODE") print "STAGEOUT IS USING NEW STAGEOUT CODE" manager = WMCore.Storage.FileManager.StageOutMgr( retryPauseTime = self.step.retryDelay, numberOfRetries = self.step.retryCount, **stageOutCall) # We need to find a list of steps in our task # And eventually a list of jobReports for out steps # Search through steps for report files filesTransferred = [] for step in self.stepSpace.taskSpace.stepSpaces(): if step == self.stepName: #Don't try to parse your own report; it's not there yet continue stepLocation = os.path.join(self.stepSpace.taskSpace.location, step) logging.info("Beginning report processing for step %s" % (step)) reportLocation = os.path.join(stepLocation, 'Report.pkl') if not os.path.isfile(reportLocation): logging.error("Cannot find report for step %s in space %s" \ % (step, stepLocation)) continue # First, get everything from a file and 'unpersist' it stepReport = Report() stepReport.unpersist(reportLocation, step) taskID = getattr(stepReport.data, 'id', None) # Don't stage out files from bad steps. if not stepReport.stepSuccessful(step): continue # Okay, time to start using stuff # Now I'm a bit confused about this; each report should ONLY # Have the results of that particular step in it, # So getting all the files should get ONLY the files # for that step; or so I hope files = stepReport.getAllFileRefsFromStep(step = step) for file in files: if not hasattr(file, 'lfn') and hasattr(file, 'pfn'): # Then we're truly hosed on this file; ignore it msg = "Not a file: %s" % file logging.error(msg) continue # Support direct-to-merge # This requires pulling a bunch of stuff from everywhere # First check if it's needed if hasattr(self.step.output, 'minMergeSize') \ and hasattr(file, 'size') \ and not getattr(file, 'merged', False): # We need both of those to continue, and we don't # direct-to-merge if getattr(self.step.output, 'doNotDirectMerge', False): # Then we've been told explicitly not to do direct-to-merge continue if file.size >= self.step.output.minMergeSize: # Then this goes direct to merge try: file = self.handleLFNForMerge(mergefile = file, step = step) except Exception as ex: logging.error("Encountered error while handling LFN for merge due to size.\n") logging.error(str(ex)) logging.debug(file) logging.debug("minMergeSize: %s" % self.step.output.minMergeSize) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60401, "DirectToMergeFailure", str(ex)) elif getattr(self.step.output, 'maxMergeEvents', None) != None\ and getattr(file, 'events', None) != None\ and not getattr(file, 'merged', False): # Then direct-to-merge due to events if # the file is large enough: if file.events >= self.step.output.maxMergeEvents: # straight to merge try: file = self.handleLFNForMerge(mergefile = file, step = step) except Exception as ex: logging.error("Encountered error while handling LFN for merge due to events.\n") logging.error(str(ex)) logging.debug(file) logging.debug("maxMergeEvents: %s" % self.step.output.maxMergeEvents) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60402, "DirectToMergeFailure", str(ex)) # Save the input PFN in case we need it # Undecided whether to move file.pfn to the output PFN file.InputPFN = file.pfn lfn = getattr(file, 'lfn') fileSource = getattr(file, 'Source', None) if fileSource in ['TFileService', 'UserDefined']: userLfnRegEx(lfn) else: lfnRegEx(lfn) fileForTransfer = {'LFN': lfn, 'PFN': getattr(file, 'pfn'), 'SEName' : None, 'PNN' : None, 'StageOutCommand': None, 'Checksums' : getattr(file, 'checksums', None)} signal.signal(signal.SIGALRM, alarmHandler) signal.alarm(waitTime) try: manager(fileForTransfer) #Afterwards, the file should have updated info. filesTransferred.append(fileForTransfer) file.StageOutCommand = fileForTransfer['StageOutCommand'] # file.location = fileForTransfer['SEName'] file.location = fileForTransfer['PNN'] file.OutputPFN = fileForTransfer['PFN'] except Alarm: msg = "Indefinite hang during stageOut of logArchive" logging.error(msg) manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60403, "StageOutTimeout", msg) stepReport.persist("Report.pkl") except Exception as ex: manager.cleanSuccessfulStageOuts() stepReport.addError(self.stepName, 60307, "StageOutFailure", str(ex)) stepReport.setStepStatus(self.stepName, 1) stepReport.persist("Report.pkl") raise signal.alarm(0) # Am DONE with report # Persist it stepReport.persist(reportLocation) #Done with all steps, and should have a list of #stagedOut files in fileForTransfer logging.info("Transferred %i files" %(len(filesTransferred))) return