def testUnitTestBackendNew(self):
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
helper = StageOutTemplate.StageOutStepHelper(self.stepdata)
helper.addOverride(override='command', overrideValue='test-win')
helper.addOverride(override='option', overrideValue='')
helper.addOverride(override='phedex-node',
overrideValue='charlie.sheen.biz')
helper.addOverride(override='lfn-prefix', overrideValue='test-win')
helper.setNewStageoutOverride(True)
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute()
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
def testCPBackendStageOutAgainstReportNew(self):
myReport = Report()
reportPath = os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace',
'cmsRun1', 'Report.pkl')
myReport.unpersist(reportPath)
myReport.data.cmsRun1.status = 0
myReport.persist(reportPath)
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newStageOut = True
executor.step = self.stepdata
# It should fail with:
# AssertionError: LFN candidate: hosts doesn't match any of the following regular expressions:
with self.assertRaises(AssertionError):
executor.execute()
# now fix those output file names to pass the Lexicon check, and execute it again
myReport.unpersist(reportPath)
# cmsRun1.output.FEVT.files.file0.lfn = 'hosts'
# cmsRun1.output.ALCARECOStreamCombined.files.file0.lfn = '/test1/hosts'
myReport.data.cmsRun1.output.FEVT.files.file0.lfn = "/store/mc/acqera/pd/FEVT/procstr/abc123.root"
myReport.data.cmsRun1.output.ALCARECOStreamCombined.files.file0.lfn = "/store/mc/acqera/pd/ALCARECO/procstr/abc123.root"
myReport.persist(reportPath)
executor.execute()
self.assertTrue(
os.path.exists(
os.path.join(self.testDir, "store", "mc", "acqera", "pd",
"FEVT")))
self.assertTrue(
os.path.exists(
os.path.join(self.testDir, "store", "mc", "acqera", "pd",
"ALCARECO")))
def testCPBackendStageOutAgainstReportNew(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 0
myReport.persist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
self.stepdata.override.newStageOut = True
executor.step = self.stepdata
executor.execute( )
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertTrue( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
def testExecutorDoesntDetonate(self):
myReport = Report()
myReport.unpersist(os.path.join( self.testDir,'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
myReport.data.cmsRun1.status = 1
myReport.persist(os.path.join( self.testDir, 'UnitTests','WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise( self.stepdata, self.job)
self.setLocalOverride(self.stepdata)
executor.step = self.stepdata
executor.execute( )
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'hosts' )))
self.assertFalse( os.path.exists( os.path.join( self.testDir, 'test1', 'hosts')))
return
def testOnWorkerNodes(self):
raise RuntimeError
# Stage a file out, stage it back in, check it, delete it
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
myReport.data.cmsRun1.status = 1
del myReport.data.cmsRun1.output
myReport.data.cmsRun1.section_('output')
myReport.data.cmsRun1.output.section_('stagingTestOutput')
myReport.data.cmsRun1.output.stagingTestOutput.section_('files')
myReport.data.cmsRun1.output.stagingTestOutput.fileCount = 0
targetFiles = [
'/store/temp/WMAgent/storetest-%s' % time.time(),
'/store/unmerged/WMAgent/storetest-%s' % time.time()
]
for file in targetFiles:
print("Adding file for StageOut %s" % file)
self.addStageOutFile(myReport, file)
myReport.persist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
executor = StageOutExecutor.StageOut()
executor.initialise(self.stepdata, self.job)
executor.step = self.stepdata
print("beginning stageout")
executor.execute()
print("stageout done")
# pull in the report with the stage out info
myReport = Report()
myReport.unpersist(
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
print("Got the stage out data back")
print(myReport.data)
# now, transfer them back
# TODO make a stagein step in the task - Melo
import WMCore.Storage.FileManager as FileManagerModule
fileManager = FileManagerModule.FileManager(numberOfRetries=10,
retryPauseTime=1)
for file in targetFiles:
print("Staging in %s" % file)
fileManager.stageOut(fileToStage={
'LFN': file,
'PFN': '%s/%s' % (self.testDir, file)
})
self.assertTrue(os.path.exists('%s/%s' % (self.testDir, file)))
# self.assertEqual(os.path.getsize('/etc/hosts', '%s/%s' % (self.testDir, file))) This makes no sense - EWV
# now, should delete the files we made
for file in targetFiles:
print("deleting %s" % file)
fileManager.deleteLFN(file)
# try staging in again to make sure teh files are gone
for file in targetFiles:
print("Staging in (should fail) %s" % file)
self.assertRaises(StageOutError,
FileManagerModule.FileManager.stageOut,
fileManager,
fileToStage={
'LFN': file,
'PFN': '%s/%s' % (self.testDir, file)
},
stageOut=False)
# need to make sure files didn't show up
self.assertFalse(os.path.exists(os.path.join(self.testDir, 'hosts')))
self.assertFalse(
os.path.exists(os.path.join(self.testDir, 'test1', 'hosts')))
