def testUnitTestBackendNew(self): myReport = Report() myReport.unpersist( os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl')) myReport.data.cmsRun1.status = 1 myReport.persist( os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl')) executor = StageOutExecutor.StageOut() helper = StageOutTemplate.StageOutStepHelper(self.stepdata) helper.addOverride(override='command', overrideValue='test-win') helper.addOverride(override='option', overrideValue='') helper.addOverride(override='phedex-node', overrideValue='charlie.sheen.biz') helper.addOverride(override='lfn-prefix', overrideValue='test-win') helper.setNewStageoutOverride(True) executor.initialise(self.stepdata, self.job) self.setLocalOverride(self.stepdata) executor.step = self.stepdata executor.execute() self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts'))) self.assertFalse( os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
def testCPBackendStageOutAgainstReportNew(self): myReport = Report() reportPath = os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl') myReport.unpersist(reportPath) myReport.data.cmsRun1.status = 0 myReport.persist(reportPath) executor = StageOutExecutor.StageOut() executor.initialise(self.stepdata, self.job) self.setLocalOverride(self.stepdata) self.stepdata.override.newStageOut = True executor.step = self.stepdata # It should fail with: # AssertionError: LFN candidate: hosts doesn't match any of the following regular expressions: with self.assertRaises(AssertionError): executor.execute() # now fix those output file names to pass the Lexicon check, and execute it again myReport.unpersist(reportPath) # cmsRun1.output.FEVT.files.file0.lfn = 'hosts' # cmsRun1.output.ALCARECOStreamCombined.files.file0.lfn = '/test1/hosts' myReport.data.cmsRun1.output.FEVT.files.file0.lfn = "/store/mc/acqera/pd/FEVT/procstr/abc123.root" myReport.data.cmsRun1.output.ALCARECOStreamCombined.files.file0.lfn = "/store/mc/acqera/pd/ALCARECO/procstr/abc123.root" myReport.persist(reportPath) executor.execute() self.assertTrue( os.path.exists( os.path.join(self.testDir, "store", "mc", "acqera", "pd", "FEVT"))) self.assertTrue( os.path.exists( os.path.join(self.testDir, "store", "mc", "acqera", "pd", "ALCARECO")))
def testCPBackendStageOutAgainstReportNew(self): myReport = Report() myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl')) myReport.data.cmsRun1.status = 0 myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl')) executor = StageOutExecutor.StageOut() executor.initialise( self.stepdata, self.job) self.setLocalOverride(self.stepdata) self.stepdata.override.newStageOut = True executor.step = self.stepdata executor.execute( ) self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' ))) self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
def testExecutorDoesntDetonate(self): myReport = Report() myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl')) myReport.data.cmsRun1.status = 1 myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl')) executor = StageOutExecutor.StageOut() executor.initialise( self.stepdata, self.job) self.setLocalOverride(self.stepdata) executor.step = self.stepdata executor.execute( ) self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' ))) self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts'))) return
def testOnWorkerNodes(self): raise RuntimeError # Stage a file out, stage it back in, check it, delete it myReport = Report() myReport.unpersist( os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl')) myReport.data.cmsRun1.status = 1 del myReport.data.cmsRun1.output myReport.data.cmsRun1.section_('output') myReport.data.cmsRun1.output.section_('stagingTestOutput') myReport.data.cmsRun1.output.stagingTestOutput.section_('files') myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0 targetFiles = [ '/store/temp/WMAgent/storetest-%s' % time.time(), '/store/unmerged/WMAgent/storetest-%s' % time.time() ] for file in targetFiles: print("Adding file for StageOut %s" % file) self.addStageOutFile(myReport, file) myReport.persist( os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl')) executor = StageOutExecutor.StageOut() executor.initialise(self.stepdata, self.job) executor.step = self.stepdata print("beginning stageout") executor.execute() print("stageout done") # pull in the report with the stage out info myReport = Report() myReport.unpersist( os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl')) print("Got the stage out data back") print(myReport.data) # now, transfer them back # TODO make a stagein step in the task - Melo import WMCore.Storage.FileManager as FileManagerModule fileManager = FileManagerModule.FileManager(numberOfRetries=10, retryPauseTime=1) for file in targetFiles: print("Staging in %s" % file) fileManager.stageOut(fileToStage={ 'LFN': file, 'PFN': '%s/%s' % (self.testDir, file) }) self.assertTrue(os.path.exists('%s/%s' % (self.testDir, file))) # self.assertEqual(os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file))) This makes no sense - EWV # now, should delete the files we made for file in targetFiles: print("deleting %s" % file) fileManager.deleteLFN(file) # try staging in again to make sure teh files are gone for file in targetFiles: print("Staging in (should fail) %s" % file) self.assertRaises(StageOutError, FileManagerModule.FileManager.stageOut, fileManager, fileToStage={ 'LFN': file, 'PFN': '%s/%s' % (self.testDir, file) }, stageOut=False) # need to make sure files didn't show up self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts'))) self.assertFalse( os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))