def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workloadDir = os.path.join(self.testDir, workloadName)
#arguments = getTestArguments()
#workload = rerecoWorkload("Tier1ReReco", arguments)
#rereco = workload.getTask("ReReco")
workload = testWorkload(emulation = emulator)
rereco = workload.getTask("ReReco")
# Set environment and site-local-config
siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
if not os.path.exists(siteConfigPath):
os.makedirs(siteConfigPath)
shutil.copy('site-local-config.xml', siteConfigPath)
environment = rereco.data.section_('environment')
environment.CMS_PATH = workloadDir
taskMaker = TaskMaker(workload, workloadDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workloadDir = os.path.join(self.testDir, workloadName)
#arguments = getTestArguments()
#workload = rerecoWorkload("Tier1ReReco", arguments)
#rereco = workload.getTask("ReReco")
workload = testWorkload(emulation = emulator)
rereco = workload.getTask("ReReco")
# Set environment and site-local-config
siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/')
if not os.path.exists(siteConfigPath):
os.makedirs(siteConfigPath)
shutil.copy('site-local-config.xml', siteConfigPath)
environment = rereco.data.section_('environment')
environment.CMS_PATH = workloadDir
taskMaker = TaskMaker(workload, workloadDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def main():
"""main functionf for testing"""
from WMCore.DataStructs.Job import Job
from WMCore.DataStructs.File import File
from WMCore.DataStructs.Run import Run
from WMCore.DataStructs.JobPackage import JobPackage
from WMCore.Services.UUID import makeUUID
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
factory = HarvestingWorkloadFactory()
workload = factory("derp", getTestArguments())
task = workload.getTask('Harvesting')
job = Job("SampleJob")
job["id"] = makeUUID()
job["task"] = task.getPathName()
job["workflow"] = workload.name()
file = File(lfn="/store/relval/CMSSW_3_8_2/RelValMinBias/GEN-SIM-RECO/MC_38Y_V9-v1/0019/FEC5BB4D-BFAF-DF11-A52A-001A92810AD2.root")
job.addFile(file)
jpackage = JobPackage()
jpackage[1] = job
import pickle
handle = open("%s/JobPackage.pkl" % os.getcwd(), 'w')
pickle.dump(jpackage, handle)
handle.close()
taskMaker = TaskMaker(workload, os.getcwd())
taskMaker.skipSubscription = True
taskMaker.processWorkload()
task.build(os.getcwd())
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def createTestWorkload(self, workloadName = 'Tier1ReReco'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createTestWorkload(self, workloadName = 'Tier1ReReco'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createTestWorkload(self, workloadName="Test", emulator=True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, "workloadTest"))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createTestWorkload(self, name='workloadTest'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload()
taskMaker = TaskMaker(workload, os.path.join(self.testDir, name))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.workloadSpecPath = os.path.join(self.testDir, name,
"%s/WMSandbox/WMWorkload.pkl" % name)
return workload
def createTestWorkload(self):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload()
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.workloadSpecPath = os.path.join(self.testDir, 'workloadTest',
"TestWorkload/WMSandbox/WMWorkload.pkl")
return workload
def createWorkload(self, workloadName='Test'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
rereco = workload.getTask("ReReco")
seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001}
rereco.addGenerator("PresetSeeder", **seederDict)
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createWorkload(self, workloadName='Test', emulator=True, priority=1):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
rereco = workload.getTask("ReReco")
seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001}
rereco.addGenerator("PresetSeeder", **seederDict)
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.setCampaign(self.campaignName)
workload.save(workloadName)
return workload
def createTestWorkload(self, workloadName='Test', emulator=True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload,
os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def createWorkload(self, workloadName='Test'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
# Add RequestManager stuff
workload.data.request.section_('schema')
workload.data.request.schema.Requestor = 'nobody'
workload.data.request.schema.Group = 'testers'
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createWorkload(self, workloadName='Test'):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
# Add RequestManager stuff
workload.data.request.section_('schema')
workload.data.request.schema.Requestor = 'nobody'
workload.data.request.schema.Group = 'testers'
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createWorkload(self, workloadName):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload(workloadName)
taskMaker = TaskMaker(workload,
os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.setCampaign(self.campaignName)
workload.save(workloadName)
return workload
def createWorkload(self, workloadName="Test", emulator=True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("Tier1ReReco")
rereco = workload.getTask("ReReco")
# Add RequestManager stuff
workload.data.request.section_("schema")
workload.data.request.schema.Requestor = "nobody"
workload.data.request.schema.Group = "testers"
taskMaker = TaskMaker(workload, os.path.join(self.testDir, "workloadTest"))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
return workload
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
#arguments = getTestArguments()
#workload = rerecoWorkload("Tier1ReReco", arguments)
workload = testWorkload("Tier1ReReco")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)
myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
socketLoc = socketLoc)
workloadName = "CmsRunAnalysis-%s" % arguments["ProcessingVersion"]
workloadFile = "CmsRunAnalysis-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
cmsRunAna = AnalysisWorkloadFactory()
workload = cmsRunAna(workloadName, arguments)
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(os.path.join(workloadName, workloadFile))
def doIndent(level):
myStr = ""
while level > 0:
myStr = myStr + " "
level -= 1
return myStr
def injectTaskIntoWMBS(specUrl, workflowName, task, inputFileset, indent = 0):
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join( self.testDir, 'cmsRun1')
os.mkdir( self.cmsstepdir )
open( os.path.join( self.cmsstepdir, '__init__.py'),'w').close()
open( os.path.join( self.cmsstepdir, 'Report.pkl'),'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder( cmsstep.data, 'Production', self.cmsstepdir )
realstep = LogArchiveTemplate.LogArchiveStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join( self.testDir, 'stepdir')
os.mkdir( self.stepDir )
builder = LogArchiveBuilder.LogArchive()
builder( step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('LogArchive')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close()
shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'),
os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
os.mkdir(self.cmsstepdir)
open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
realstep = StageOutTemplate.StageOutStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join(self.testDir, 'stepdir')
os.mkdir(self.stepDir)
builder = StageOutBuilder.StageOut()
builder(step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
