def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def setUp(self): # stolen from CMSSWExecutor_t. thanks, dave self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") stepHelper = step = self.task.makeStep("DeleteTest") self.step = stepHelper.data self.actualStep = stepHelper template = DeleteTemplate() template(self.step) self.helper = template.helper(self.step) self.executor = StepFactory.getStepExecutor(self.actualStep.stepType()) taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.insert(0, self.testDir) sys.path.insert(0, self.sandboxDir) self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def createTestWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workloadDir = os.path.join(self.testDir, workloadName) #arguments = getTestArguments() #workload = rerecoWorkload("Tier1ReReco", arguments) #rereco = workload.getTask("ReReco") workload = testWorkload(emulation = emulator) rereco = workload.getTask("ReReco") # Set environment and site-local-config siteConfigPath = os.path.join(workloadDir, 'SITECONF/local/JobConfig/') if not os.path.exists(siteConfigPath): os.makedirs(siteConfigPath) shutil.copy('site-local-config.xml', siteConfigPath) environment = rereco.data.section_('environment') environment.CMS_PATH = workloadDir taskMaker = TaskMaker(workload, workloadDir) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload
def main(): """main functionf for testing""" from WMCore.DataStructs.Job import Job from WMCore.DataStructs.File import File from WMCore.DataStructs.Run import Run from WMCore.DataStructs.JobPackage import JobPackage from WMCore.Services.UUID import makeUUID from WMCore.WMSpec.Makers.TaskMaker import TaskMaker factory = HarvestingWorkloadFactory() workload = factory("derp", getTestArguments()) task = workload.getTask('Harvesting') job = Job("SampleJob") job["id"] = makeUUID() job["task"] = task.getPathName() job["workflow"] = workload.name() file = File(lfn="/store/relval/CMSSW_3_8_2/RelValMinBias/GEN-SIM-RECO/MC_38Y_V9-v1/0019/FEC5BB4D-BFAF-DF11-A52A-001A92810AD2.root") job.addFile(file) jpackage = JobPackage() jpackage[1] = job import pickle handle = open("%s/JobPackage.pkl" % os.getcwd(), 'w') pickle.dump(jpackage, handle) handle.close() taskMaker = TaskMaker(workload, os.getcwd()) taskMaker.skipSubscription = True taskMaker.processWorkload() task.build(os.getcwd())
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def createTestWorkload(self, workloadName = 'Tier1ReReco'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createTestWorkload(self, workloadName="Test", emulator=True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, "workloadTest")) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createTestWorkload(self, name='workloadTest'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload() taskMaker = TaskMaker(workload, os.path.join(self.testDir, name)) taskMaker.skipSubscription = True taskMaker.processWorkload() self.workloadSpecPath = os.path.join(self.testDir, name, "%s/WMSandbox/WMWorkload.pkl" % name) return workload
def createTestWorkload(self): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload() taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() self.workloadSpecPath = os.path.join(self.testDir, 'workloadTest', "TestWorkload/WMSandbox/WMWorkload.pkl") return workload
def createWorkload(self, workloadName='Test'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) rereco = workload.getTask("ReReco") seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001} rereco.addGenerator("PresetSeeder", **seederDict) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createWorkload(self, workloadName='Test', emulator=True, priority=1): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001} rereco.addGenerator("PresetSeeder", **seederDict) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.setCampaign(self.campaignName) workload.save(workloadName) return workload
def createTestWorkload(self, workloadName='Test', emulator=True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload
def createWorkload(self, workloadName='Test'): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) # Add RequestManager stuff workload.data.request.section_('schema') workload.data.request.schema.Requestor = 'nobody' workload.data.request.schema.Group = 'testers' taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createWorkload(self, workloadName): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload(workloadName) taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.setCampaign(self.campaignName) workload.save(workloadName) return workload
def createWorkload(self, workloadName="Test", emulator=True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") # Add RequestManager stuff workload.data.request.section_("schema") workload.data.request.schema.Requestor = "nobody" workload.data.request.schema.Group = "testers" taskMaker = TaskMaker(workload, os.path.join(self.testDir, "workloadTest")) taskMaker.skipSubscription = True taskMaker.processWorkload() return workload
def createTestWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ #arguments = getTestArguments() #workload = rerecoWorkload("Tier1ReReco", arguments) workload = testWorkload("Tier1ReReco") rereco = workload.getTask("ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None) connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None) (dialect, junk) = connectUrl.split(":", 1) myWMInit = WMInit() myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect, socketLoc = socketLoc) workloadName = "CmsRunAnalysis-%s" % arguments["ProcessingVersion"] workloadFile = "CmsRunAnalysis-%s.pkl" % arguments["ProcessingVersion"] os.mkdir(workloadName) cmsRunAna = AnalysisWorkloadFactory() workload = cmsRunAna(workloadName, arguments) taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName)) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(os.path.join(workloadName, workloadFile)) def doIndent(level): myStr = "" while level > 0: myStr = myStr + " " level -= 1 return myStr def injectTaskIntoWMBS(specUrl, workflowName, task, inputFileset, indent = 0):
def setUp(self): self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # shut up SiteLocalConfig os.environ['CMS_PATH'] = os.getcwd() workload = copy.deepcopy(testWorkloads.workload) task = workload.getTask("Production") step = task.getStep("stageOut1") # want to get the cmsstep so I can make the Report cmsstep = task.getStep('cmsRun1') self.cmsstepdir = os.path.join( self.testDir, 'cmsRun1') os.mkdir( self.cmsstepdir ) open( os.path.join( self.cmsstepdir, '__init__.py'),'w').close() open( os.path.join( self.cmsstepdir, 'Report.pkl'),'w').close() cmsbuilder = CMSSWBuilder.CMSSW() cmsbuilder( cmsstep.data, 'Production', self.cmsstepdir ) realstep = LogArchiveTemplate.LogArchiveStepHelper(step.data) realstep.disableRetries() self.realstep = realstep self.stepDir = os.path.join( self.testDir, 'stepdir') os.mkdir( self.stepDir ) builder = LogArchiveBuilder.LogArchive() builder( step.data, 'Production', self.stepDir) # stolen from CMSSWExecutor_t. thanks, dave # first, delete all the sandboxen and taskspaces # because of caching, this leaks from other tests in other files # this sucks because the other tests are using sandboxen that # are deleted after the test is over, which causes theses tests # to break modsToDelete = [] # not sure what happens if you delete from # an arrey you're iterating over. doing it in # two steps for modname in sys.modules.keys(): # need to blow away things in sys.modules, otherwise # they are cached and we look at old taskspaces if modname.startswith('WMTaskSpace'): modsToDelete.append(modname) if modname.startswith('WMSandbox'): modsToDelete.append(modname) for modname in modsToDelete: try: reload(sys.modules[modname]) except: pass del sys.modules[modname] self.oldpath = sys.path[:] self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.job = Job(name = "/UnitTests/DeleterTask/DeleteTest-test-job") shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") cmsswHelper = self.task.makeStep("cmsRun1") cmsswHelper.setStepType('CMSSW') stepHelper = cmsswHelper.addStep("DeleteTest") stepHelper.setStepType('LogArchive') self.cmsswstep = cmsswHelper.data self.cmsswHelper = cmsswHelper self.stepdata = stepHelper.data self.stephelp = LogArchiveTemplate.LogArchiveStepHelper(stepHelper.data) self.task.applyTemplates() self.executor = StepFactory.getStepExecutor(self.stephelp.stepType()) taskMaker = TaskMaker(self.workload, os.path.join(self.testDir)) taskMaker.skipSubscription = True taskMaker.processWorkload() self.task.build(os.path.join(self.testDir, 'UnitTests')) sys.path.insert(0, self.testDir) sys.path.insert(0, os.path.join(self.testDir, 'UnitTests')) # binDir = inspect.getsourcefile(ModuleLocator) # binDir = binDir.replace("__init__.py", "bin") # # if not binDir in os.environ['PATH']: # os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) open( os.path.join( self.testDir, 'UnitTests', '__init__.py'),'w').close() shutil.copyfile( os.path.join( os.path.dirname( __file__ ), 'MergeSuccess.pkl'), os.path.join( self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1' , 'Report.pkl'))
def setUp(self): self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # shut up SiteLocalConfig os.environ['CMS_PATH'] = os.getcwd() workload = copy.deepcopy(testWorkloads.workload) task = workload.getTask("Production") step = task.getStep("stageOut1") # want to get the cmsstep so I can make the Report cmsstep = task.getStep('cmsRun1') self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1') os.mkdir(self.cmsstepdir) open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close() open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close() cmsbuilder = CMSSWBuilder.CMSSW() cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir) realstep = StageOutTemplate.StageOutStepHelper(step.data) realstep.disableRetries() self.realstep = realstep self.stepDir = os.path.join(self.testDir, 'stepdir') os.mkdir(self.stepDir) builder = StageOutBuilder.StageOut() builder(step.data, 'Production', self.stepDir) # stolen from CMSSWExecutor_t. thanks, dave # first, delete all the sandboxen and taskspaces # because of caching, this leaks from other tests in other files # this sucks because the other tests are using sandboxen that # are deleted after the test is over, which causes theses tests # to break modsToDelete = [] # not sure what happens if you delete from # an arrey you're iterating over. doing it in # two steps for modname in sys.modules.keys(): # need to blow away things in sys.modules, otherwise # they are cached and we look at old taskspaces if modname.startswith('WMTaskSpace'): modsToDelete.append(modname) if modname.startswith('WMSandbox'): modsToDelete.append(modname) for modname in modsToDelete: try: reload(sys.modules[modname]) except Exception: pass del sys.modules[modname] self.oldpath = sys.path[:] self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job") shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") cmsswHelper = self.task.makeStep("cmsRun1") cmsswHelper.setStepType('CMSSW') stepHelper = cmsswHelper.addStep("DeleteTest") stepHelper.setStepType('StageOut') self.cmsswstep = cmsswHelper.data self.cmsswHelper = cmsswHelper self.stepdata = stepHelper.data self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data) self.task.applyTemplates() self.executor = StepFactory.getStepExecutor(self.stephelp.stepType()) taskMaker = TaskMaker(self.workload, os.path.join(self.testDir)) taskMaker.skipSubscription = True taskMaker.processWorkload() self.task.build(os.path.join(self.testDir, 'UnitTests')) sys.path.insert(0, self.testDir) sys.path.insert(0, os.path.join(self.testDir, 'UnitTests')) # binDir = inspect.getsourcefile(ModuleLocator) # binDir = binDir.replace("__init__.py", "bin") # # if not binDir in os.environ['PATH']: # os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) open(os.path.join(self.testDir, 'UnitTests', '__init__.py'), 'w').close() shutil.copyfile( os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'), os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl'))