def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
def setUp(self): """bootstrap tests""" self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() #self.testInit.setDatabaseConnection() #self.testInit.setSchema(customModules = ["WMCore.WMBS"], # useDefault = False) self.testInit.setupCouch("wmcore-workloadsummary", "WorkloadSummary") self.workload1 = newWorkload("WorkloadSummaryTest1") self.workload2 = newWorkload("WorkloadSummaryTest2")
def createWorkload(self): """ _createWorkload_ Create a new workload. """ ownerProps = { 'dn': self.owner_dn, 'vogroup': self.owner_vogroup, 'vorole': self.owner_vorole } workload = newWorkload(self.workloadName) workload.setOwnerDetails(self.owner, self.group, ownerProps) workload.setStartPolicy("DatasetBlock", SliceType="NumberOfFiles", SliceSize=1) workload.setEndPolicy("SingleShot") workload.setAcquisitionEra(acquisitionEras=self.acquisitionEra) workload.setProcessingVersion( processingVersions=self.processingVersion) workload.setProcessingString(processingStrings=self.processingString) workload.setValidStatus(validStatus=self.validStatus) workload.setLumiList(lumiLists=self.lumiList) workload.setPriority(self.priority) workload.setPrepID(self.prepID) return workload
def createTestWorkload(self): """ _createTestWorkload_ """ workload = newWorkload("ACDCTest") reco = workload.newTask("reco") skim1 = reco.addTask("skim1") workload.setOwnerDetails(name="evansde77", group="DMWM") # first task uses the input dataset reco.addInputDataset(name='/PRIMARY/processed-v1/TIERONE', primary="PRIMARY", processed="processed-v1", tier="TIERONE") cmsRunReco = reco.makeStep("cmsRun1") cmsRunReco.setStepType("CMSSW") reco.applyTemplates() cmsRunRecoHelper = cmsRunReco.getTypeHelper() cmsRunRecoHelper.addOutputModule("outputRECO", primaryDataset="PRIMARY", processedDataset="processed-v2", dataTier="TIERTWO", lfnBase="/store/dunkindonuts", mergedLFNBase="/store/kfc") # second step uses an input reference cmsRunSkim = skim1.makeStep("cmsRun2") cmsRunSkim.setStepType("CMSSW") skim1.applyTemplates() skim1.setInputReference(cmsRunReco, outputModule="outputRECO") return workload
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def testB(self): """ try using the helper API to set and get information in the step """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) try: helper = template.helper(step) except Exception as ex: msg = "Failure to create CMSSW Step Helper" msg += str(ex) self.fail(msg) helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443") helper.addOutputModule( "outputModule1", primaryDataset="Primary", processedDataset='Processed', dataTier='Tier', lfnBase="/store/unmerged/whatever" )
def testA(self): """ instantiate & apply template """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data # first up, create the template try: template = CMSSWTemplate() except Exception as ex: msg = "Failed to instantiate the CMSSW Step Template" msg += str(ex) self.fail(msg) # now apply it to the step try: template(step) except Exception as ex: msg = "Failed to apply template to step" msg += str(ex) self.fail(msg) # TODO: Check the step has the appropriate attributes expected for CMSSW self.assertTrue( hasattr(step, "application")) self.assertTrue( hasattr(step.application, "setup"))
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def setUp(self): # stolen from CMSSWExecutor_t. thanks, dave self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") stepHelper = step = self.task.makeStep("DeleteTest") self.step = stepHelper.data self.actualStep = stepHelper template = DeleteTemplate() template(self.step) self.helper = template.helper(self.step) self.executor = StepFactory.getStepExecutor(self.actualStep.stepType()) taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.insert(0, self.testDir) sys.path.insert(0, self.sandboxDir) self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setupRepackWorkflow(self): """ _setupRepackWorkflow_ Populate WMBS with a repack-like workflow, every subscription must be unfinished at first """ workflowName = 'Repack_Run481516_StreamZ' mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW', 'RepackMergewrite_SingleNeutrino_RAW'] self.stateMap = {'Merge' : [], 'Processing Done' : []} self.orderedStates = ['Merge', 'Processing Done'] # Populate WMStats self.wmstatsWriter.insertGenericRequest({'_id' : workflowName}) self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) repackTask = workload.newTask('Repack') for task in mergeTasks: repackTask.addTask(task) repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW') specPath = os.path.join(self.testDir, 'Repack.pkl') workload.save(specPath) # Populate WMBS topFileset = Fileset(name = 'TestStreamerFileset') topFileset.create() options = {'spec' : specPath, 'owner' : 'ItsAMeMario', 'name' : workflowName, 'wfType' : 'tier0'} topLevelWorkflow = Workflow(task = '/%s/Repack' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(topFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topFileset) for task in mergeTasks: mergeWorkflow = Workflow(task = '/%s/Repack/%s' % (workflowName, task), **options) mergeWorkflow.create() unmergedFileset = Fileset(name = 'TestUnmergedFileset%s' % task) unmergedFileset.create() mergeSub = Subscription(unmergedFileset, mergeWorkflow) mergeSub.create() self.stateMap['Processing Done'].append(unmergedFileset) cleanupWorkflow = Workflow(task = '/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW', **options) cleanupWorkflow.create() unmergedFileset = Fileset(name = 'TestUnmergedFilesetToCleanup') unmergedFileset.create() cleanupSub = Subscription(unmergedFileset, cleanupWorkflow) cleanupSub.create() return
def setupRepackWorkflow(self): """ _setupRepackWorkflow_ Populate WMBS with a repack-like workflow, every subscription must be unfinished at first """ workflowName = 'Repack_Run481516_StreamZ' mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW', 'RepackMergewrite_SingleNeutrino_RAW'] self.stateMap = {'Merge': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) repackTask = workload.newTask('Repack') for task in mergeTasks: repackTask.addTask(task) repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW') specPath = os.path.join(self.testDir, 'Repack.pkl') workload.save(specPath) # Populate WMBS topFileset = Fileset(name='TestStreamerFileset') topFileset.create() options = {'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0'} topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(topFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topFileset) for task in mergeTasks: mergeWorkflow = Workflow(task='/%s/Repack/%s' % (workflowName, task), **options) mergeWorkflow.create() unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task) unmergedFileset.create() mergeSub = Subscription(unmergedFileset, mergeWorkflow) mergeSub.create() self.stateMap['Processing Done'].append(unmergedFileset) cleanupWorkflow = Workflow(task='/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW', **options) cleanupWorkflow.create() unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup') unmergedFileset.create() cleanupSub = Subscription(unmergedFileset, cleanupWorkflow) cleanupSub.create() return
def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals( self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = { 'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_file': 5 } self.assertEquals( self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema')
def createWorkload(name="BasicProcessing"): workload = newWorkload(name) workload.setOwner("DMWMTest") workload.setStartPolicy('DatasetBlock') workload.setEndPolicy('SingleShot') workload.setCampaign("TestCampaign") workload.setPrepID("TestPrepID") # // # // set up the production task #// rereco = workload.newTask("ReReco1") rereco.setTaskType("Processing") rerecoCmssw = rereco.makeStep("cmsRun1") rerecoCmssw.setStepType("CMSSW") skimStageOut = rerecoCmssw.addStep("stageOut1") skimStageOut.setStepType("StageOut") skimLogArch = rerecoCmssw.addStep("logArch1") skimLogArch.setStepType("LogArchive") rereco.applyTemplates() rereco.setSplittingAlgorithm("FileBased", files_per_job=1) rereco.addInputDataset( primary = "Cosmics", processed = "CRAFT09-PromptReco-v1", tier = "RECO", dbsurl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader") # // # // rereco cmssw step #// # # TODO: Anywhere helper.data is accessed means we need a method added to the # type based helper class to provide a clear API. rerecoCmsswHelper = rerecoCmssw.getTypeHelper() rerecoCmsswHelper.cmsswSetup( "CMSSW_3_1_2", softwareEnvironment=" . /uscmst1/prod/sw/cms/bashrc prod" ) rerecoCmsswHelper.setDataProcessingConfig( "cosmics", "promptReco", globalTag="GLOBAL::BALLS", writeTiers=['RECO']) rerecoCmsswHelper.addOutputModule( "outputRECO", primaryDataset="Primary", processedDataset="Processed", dataTier="RECO") #Add a stageOut step skimStageOutHelper = skimStageOut.getTypeHelper() skimLogArchHelper = skimLogArch.getTypeHelper() rereco.addGenerator("BasicNaming") return workload
def createWorkload(name="BasicProcessing"): workload = newWorkload(name) workload.setOwner("DMWMTest") workload.setStartPolicy('DatasetBlock') workload.setEndPolicy('SingleShot') workload.setCampaign("TestCampaign") workload.setPrepID("TestPrepID") # // # // set up the production task #// rereco = workload.newTask("ReReco1") rereco.setTaskType("Processing") rerecoCmssw = rereco.makeStep("cmsRun1") rerecoCmssw.setStepType("CMSSW") skimStageOut = rerecoCmssw.addStep("stageOut1") skimStageOut.setStepType("StageOut") skimLogArch = rerecoCmssw.addStep("logArch1") skimLogArch.setStepType("LogArchive") rereco.applyTemplates() rereco.setSplittingAlgorithm("FileBased", files_per_job=1) rereco.addInputDataset( primary = "Cosmics", processed = "CRAFT09-PromptReco-v1", tier = "RECO", dbsurl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader") # // # // rereco cmssw step #// # # TODO: Anywhere helper.data is accessed means we need a method added to the # type based helper class to provide a clear API. rerecoCmsswHelper = rerecoCmssw.getTypeHelper() rerecoCmsswHelper.cmsswSetup( "CMSSW_3_1_2", softwareEnvironment=" . /uscmst1/prod/sw/cms/bashrc prod" ) rerecoCmsswHelper.setDataProcessingConfig( "cosmics", "promptReco", globalTag="GLOBAL::BALLS", writeTiers=['RECO']) rerecoCmsswHelper.addOutputModule( "outputRECO", primaryDataset="Primary", processedDataset="Processed", dataTier="RECO") #Add a stageOut step skimStageOutHelper = skimStageOut.getTypeHelper() skimLogArchHelper = skimLogArch.getTypeHelper() rereco.addGenerator("BasicNaming") return workload
def newWorkload(self): """ _newWorkload_ Create a new workload. """ workload = newWorkload(self.workloadName) workload.setOwner(self.owner) #workload.data.properties.acquisitionEra = self.acquisitionEra return workload
def createWorkload(self): """ _createWorkload_ Create a new workload. """ workload = newWorkload("TestWorkload") workload.setOwner("*****@*****.**") workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1) workload.setEndPolicy("SingleShot") workload.setAcquisitionEra("WMAgentCommissioning10") return workload
def oneTaskTwoStep(): workload = newWorkload("OneTaskTwoStepTest") task1 = workload.newTask("FirstTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") task1.applyTemplates() return workload
def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.workflowsByStatus(["failed"]) self.assertEquals(requests, [schema[0]['RequestName']])
def oneTaskTwoStep(): workload = newWorkload("OneTaskTwoStepTest") workload.setDbsUrl("https://cmsweb.cern.ch/dbs/prod/global/DBSReader") task1 = workload.newTask("FirstTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") task1.applyTemplates() return workload
def createWorkload(self): """ _createWorkload_ Create a new workload. """ ownerProps = {'dn': self.owner_dn, 'vogroup': self.owner_vogroup, 'vorole': self.owner_vorole} workload = newWorkload(self.workloadName) workload.setOwnerDetails(self.owner, self.group, ownerProps) workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1) workload.setEndPolicy("SingleShot") workload.setAcquisitionEra(acquisitionEras = self.acquisitionEra) workload.setProcessingVersion(processingVersions = self.processingVersion) workload.setValidStatus(validStatus = self.validStatus) return workload
def testMulticoreSettings(self): """ test multicore related methods """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) helper = template.helper(step) self.assertEqual(helper.getNumberOfCores(), 1) helper.setNumberOfCores(8) self.assertEqual(helper.getNumberOfCores(), 8)
def testA(self): """ instantiate & apply template """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data # first up, create the template try: template = CMSSWTemplate() except Exception, ex: msg = "Failed to instantiate the CMSSW Step Template" msg += str(ex) self.fail(msg)
def testB(self): """ try using the helper API to set and get information in the step """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = step = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) try: helper = template.helper(step) except Exception, ex: msg = "Failure to create CMSSW Step Helper" msg += str(ex) self.fail(msg)
def testChainedProcessing(self): """ check the chained processing set up is correct """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") step = task.makeStep("TemplateTest") template = CMSSWTemplate() helper = template.helper(step.data) inputStepName = "some_inputStepName" inputOutputModule = "some_inputOutputModule" helper.setupChainedProcessing(inputStepName, inputOutputModule) self.assertEqual(helper.data.input.chainedProcessing, True) self.assertEqual(helper.data.input.inputStepName, "some_inputStepName") self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def oneTaskFourStep(): workload = newWorkload("OneTaskFourStepTest") task1 = workload.newTask("FirstTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") step3 = step1.addStep("cmsRun2") step3.setStepType("CMSSW") step4 = step3.addStep("stageOut2") step4.setStepType("StageOut") return workload
def oneTaskFourStep(): workload = newWorkload("OneTaskFourStepTest") workload.setDbsUrl("https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader") task1 = workload.newTask("FirstTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") step3 = step1.addStep("cmsRun2") step3.setStepType("CMSSW") step4 = step3.addStep("stageOut2") step4.setStepType("StageOut") return workload
def twoTaskTree(): workload = newWorkload("TwoTaskTree") task1 = workload.newTask("FirstTask") task2 = task1.addTask("SecondTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") step3 = task2.makeStep("cmsRun2") step3.setStepType("CMSSW") step4 = step3.addStep("stageOut2") step4.setStepType("StageOut") return workload
def twoTaskTree(): workload = newWorkload("TwoTaskTree") workload.setDbsUrl("https://cmsweb.cern.ch/dbs/prod/global/DBSReader") task1 = workload.newTask("FirstTask") task2 = task1.addTask("SecondTask") step1 = task1.makeStep("cmsRun1") step1.setStepType("CMSSW") step2 = step1.addStep("stageOut1") step2.setStepType("StageOut") step3 = task2.makeStep("cmsRun2") step3.setStepType("CMSSW") step4 = step3.addStep("stageOut2") step4.setStepType("StageOut") return workload
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self): """ build a step for testing purposes """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = step = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() self.sandboxDir = "%s/UnitTests" % self.testDir self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def createWorkload(self): """ _createWorkload_ Create a new workload. """ ownerProps = {'dn': self.owner_dn, 'vogroup': self.owner_vogroup, 'vorole': self.owner_vorole} workload = newWorkload(self.workloadName) workload.setOwnerDetails(self.owner, self.group, ownerProps) workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1) workload.setEndPolicy("SingleShot") workload.setAcquisitionEra(acquisitionEras = self.acquisitionEra) workload.setProcessingVersion(processingVersions = self.processingVersion) workload.setProcessingString(processingStrings = self.processingString) workload.setValidStatus(validStatus = self.validStatus) workload.setLumiList(lumiLists = self.lumiList) workload.setPriority(self.priority) workload.setCampaign(self.campaign) workload.setRequestType(self.requestType) workload.setPrepID(self.prepID) return workload
def testGPUSettings(self): """ Test GPU methods at CMSSW template level """ workload = newWorkload("UnitTests") task = workload.newTask("CMSSWTemplate") stepHelper = task.makeStep("TemplateTest") step = stepHelper.data template = CMSSWTemplate() template(step) helper = template.helper(step) self.assertEqual(helper.getGPURequired(), "forbidden") self.assertIsNone(helper.getGPURequirements()) helper.setGPUSettings("optional", "test 1 2 3") self.assertEqual(helper.getGPURequired(), "optional") self.assertItemsEqual(helper.getGPURequirements(), "test 1 2 3") helper.setGPUSettings("required", {"key1": "value1", "key2": "value2"}) self.assertEqual(helper.getGPURequired(), "required") self.assertItemsEqual(helper.getGPURequirements(), { "key1": "value1", "key2": "value2" })
def createWorkload(name="MultiTaskProduction"): workload = newWorkload("MultiTaskProduction") workload.setOwner("DMWMTest") workload.setStartPolicy('MonteCarlo') workload.setEndPolicy('SingleShot') # // # // set up the production task #// production = workload.newTask("Production1") production.addProduction(totalevents = 1000) prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1") prodStageOut.setStepType("StageOut") production.applyTemplates() production.setSiteWhitelist(["T2_XX_SiteA"]) # // # // set up the merge task #// merge = production.addTask("Merge") mergeCmssw = merge.makeStep("cmsRun1") mergeCmssw.setStepType("CMSSW") mergeStageOut = mergeCmssw.addStep("stageOut1") mergeStageOut.setStepType("StageOut") merge.applyTemplates() # // # // populate the details of the production tasks #// # // # // production cmssw step #// # # TODO: Anywhere helper.data is accessed means we need a method added to the # type based helper class to provide a clear API. prodCmsswHelper = prodCmssw.getTypeHelper() prodCmsswHelper.data.application.setup.cmsswVersion = "CMSSW_X_Y_Z" prodCmsswHelper.data.application.setup.softwareEnvironment = " . /uscmst1/prod/sw/cms/bashrc prod" prodCmsswHelper.data.application.configuration.configCacheUrl = "http://whatever" prodCmsswHelper.addOutputModule("writeData", primaryDataset = "Primary", processedDataset = "Processed", dataTier = "TIER") #print prodCmsswHelper.data # // # // production stage out step #// prodStageOutHelper = prodStageOut.getTypeHelper() # // # // merge cmssw step #// # point it at the input step from the previous task merge.setInputReference(prodCmssw, outputModule = "writeData") # // # // populate the details of the merge tasks #// # print workload.data production = workload.newTask("Production2") production.addProduction(totalevents = 2000) prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1") prodStageOut.setStepType("StageOut") production.applyTemplates() production.setSiteWhitelist(["T2_XX_SiteA"]) # // # // set up the merge task #// merge = production.addTask("Merge") mergeCmssw = merge.makeStep("cmsRun1") mergeCmssw.setStepType("CMSSW") mergeStageOut = mergeCmssw.addStep("stageOut1") mergeStageOut.setStepType("StageOut") merge.applyTemplates() # // # // populate the details of the production tasks #// # // # // production cmssw step #// # # TODO: Anywhere helper.data is accessed means we need a method added to the # type based helper class to provide a clear API. prodCmsswHelper = prodCmssw.getTypeHelper() prodCmsswHelper.data.application.setup.cmsswVersion = "CMSSW_X_Y_Z" prodCmsswHelper.data.application.setup.softwareEnvironment = " . /uscmst1/prod/sw/cms/bashrc prod" prodCmsswHelper.data.application.configuration.configCacheUrl = "http://whatever" prodCmsswHelper.addOutputModule("writeData", primaryDataset = "Primary", processedDataset = "Processed", dataTier = "TIER") #print prodCmsswHelper.data # // # // production stage out step #// prodStageOutHelper = prodStageOut.getTypeHelper() # // # // merge cmssw step #// # point it at the input step from the previous task merge.setInputReference(prodCmssw, outputModule = "writeData") return workload
def executeCMSSWStack(opts, scram): def getOutputModules(): pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\ "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\ "tweakJson = makeTweak(config.process).jsondictionary();"+\ "print tweakJson[\"process\"][\"outputModules_\"]" ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd()) if ret > 0: msg = scram.diagnostic() handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript)) mintime() sys.exit(EC_CMSRunWrapper) return literal_eval(scram.stdout) cmssw = CMSSW() cmssw.stepName = "cmsRun" cmssw.step = WMStep(cmssw.stepName) CMSSWTemplate().install(cmssw.step) cmssw.task = makeWMTask(cmssw.stepName) cmssw.workload = newWorkload(cmssw.stepName) cmssw.step.application.setup.softwareEnvironment = '' cmssw.step.application.setup.scramArch = opts.scramArch cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion cmssw.step.application.configuration.section_("arguments") cmssw.step.application.configuration.arguments.globalTag = "" for output in getOutputModules(): cmssw.step.output.modules.section_(output) getattr(cmssw.step.output.modules, output).primaryDataset = '' getattr(cmssw.step.output.modules, output).processedDataset = '' getattr(cmssw.step.output.modules, output).dataTier = '' #cmssw.step.application.command.arguments = '' #TODO cmssw.step.user.inputSandboxes = [opts.archiveJob] cmssw.step.user.userFiles = opts.userFiles or '' #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles. #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime.. cmssw.job = {'counter' : 0, 'workflow' : 'unused'} cmssw.step.user.lfnBase = '/store/temp/user/' cmssw.step.section_("builder") cmssw.step.builder.workingDir = os.getcwd() cmssw.step.runtime.invokeCommand = 'python' cmssw.step.runtime.scramPreDir = os.getcwd() cmssw.step.runtime.preScripts = [] cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+ '--inputFile=\'%s\' '+ '--runAndLumis=\'%s\' '+ '--firstEvent=%s '+ '--lastEvent=%s '+ '--firstLumi=%s '+ '--firstRun=%s '+ '--seeding=%s '+ '--lheInputFiles=%s '+ '--oneEventMode=%s ' + '--eventsPerLumi=%s') % (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis, opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun, opts.seeding, opts.lheInputFiles, opts.oneEventMode, opts.eventsPerLumi)] cmssw.step.section_("execution") #exitStatus of cmsRun is set here cmssw.report = Report("cmsRun") #report is loaded and put here cmssw.execute() return cmssw
def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.reqDBWriter.insertGenericRequest(schema[0]) self.assertEquals(result[0]['ok'], True, 'insert fail') result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed") self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned") self.assertEquals(result, 'Error: document not found') result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], {"Teams": ['teamA']}) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( "not_exist_schema", {"Teams": ['teamA']}) self.assertEquals(result, 'Error: document not found') totalStats = { 'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5 } result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( "not_exist_schema", totalStats) self.assertEquals(result, 'Error: document not found') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) properties = { "RequestPriority": spec1.priority(), 'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec1.listOutputDatasets() } result = self.reqDBWriter.updateRequestProperty( spec1.name(), properties) self.assertEquals(result, 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") properties = { "RequestPriority": spec2.priority(), 'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec2.listOutputDatasets() } result = self.reqDBWriter.updateRequestProperty( spec2.name(), properties) self.assertEquals(result, 'Error: document not found') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag=False, legacyFormat=True) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestByStatus(["failed"]) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestSummaryWithJobInfo( schema[0]['RequestName']) self.assertEquals(requests.keys(), [schema[0]['RequestName']])
_Tier1ReRecoWorkload_ """ import os, pickle, sys, shutil from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper DBSURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" # // # // Set up the basic workload task and step structure #// workload = newWorkload("MultiTaskReReco") workload.setOwner("WMTest") workload.setStartPolicy('DatasetBlock') workload.setEndPolicy('SingleShot') # // # // set up the production task #// rereco = workload.newTask("ReReco1") rerecoCmssw = rereco.makeStep("cmsRun1") rerecoCmssw.setStepType("CMSSW") skimStageOut = rerecoCmssw.addStep("stageOut1") skimStageOut.setStepType("StageOut") skimLogArch = rerecoCmssw.addStep("logArch1") skimLogArch.setStepType("LogArchive") rereco.applyTemplates()
_BasicProductionWorkload_ Sample/Tester for a simple production workflow and associated merge. Production task produces a single output dataset, Merge task is used to merge that dataset """ from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper # // # // Set up the basic workload task and step structure #// workload = newWorkload("MultiTaskProduction") workload.setOwner("WMTest") workload.setStartPolicy('MonteCarlo', **{'SliceType': 'NumberOfEvents', 'SliceSize': 100}) workload.setEndPolicy('SingleShot') # // # // set up the production task #// production = workload.newTask("Production1") production.addProduction(totalEvents = 1000) production.setTaskType("Merge") prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1") prodStageOut.setStepType("StageOut") production.applyTemplates()
Sample/Tester for a production workflow that writes several datasets and associated merges for each dataset. Production task produces three output datasets, Merge tasks are used to merge those datasets """ from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper # // # // Set up the basic workload task and step structure # // workload = newWorkload("MultiOutputs") workload.setOwner("WMTest") workload.setStartPolicy("MonteCarlo") workload.setEndPolicy("SingleShot") # // # // set up the production task # // production = workload.newTask("Production") # WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction" production.setTaskType("Merge") production.addProduction(totalEvents=1000) prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1") prodStageOut.setStepType("StageOut")
def setUp(self): self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # shut up SiteLocalConfig os.environ['CMS_PATH'] = os.getcwd() workload = copy.deepcopy(testWorkloads.workload) task = workload.getTask("Production") step = task.getStep("stageOut1") # want to get the cmsstep so I can make the Report cmsstep = task.getStep('cmsRun1') self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1') os.mkdir(self.cmsstepdir) open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close() open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close() cmsbuilder = CMSSWBuilder.CMSSW() cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir) realstep = StageOutTemplate.StageOutStepHelper(step.data) realstep.disableRetries() self.realstep = realstep self.stepDir = os.path.join(self.testDir, 'stepdir') os.mkdir(self.stepDir) builder = StageOutBuilder.StageOut() builder(step.data, 'Production', self.stepDir) # stolen from CMSSWExecutor_t. thanks, dave # first, delete all the sandboxen and taskspaces # because of caching, this leaks from other tests in other files # this sucks because the other tests are using sandboxen that # are deleted after the test is over, which causes theses tests # to break modsToDelete = [] # not sure what happens if you delete from # an arrey you're iterating over. doing it in # two steps for modname in sys.modules.keys(): # need to blow away things in sys.modules, otherwise # they are cached and we look at old taskspaces if modname.startswith('WMTaskSpace'): modsToDelete.append(modname) if modname.startswith('WMSandbox'): modsToDelete.append(modname) for modname in modsToDelete: try: reload(sys.modules[modname]) except Exception: pass del sys.modules[modname] self.oldpath = sys.path[:] self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job") shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile')) self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("DeleterTask") cmsswHelper = self.task.makeStep("cmsRun1") cmsswHelper.setStepType('CMSSW') stepHelper = cmsswHelper.addStep("DeleteTest") stepHelper.setStepType('StageOut') self.cmsswstep = cmsswHelper.data self.cmsswHelper = cmsswHelper self.stepdata = stepHelper.data self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data) self.task.applyTemplates() self.executor = StepFactory.getStepExecutor(self.stephelp.stepType()) taskMaker = TaskMaker(self.workload, os.path.join(self.testDir)) taskMaker.skipSubscription = True taskMaker.processWorkload() self.task.build(os.path.join(self.testDir, 'UnitTests')) sys.path.insert(0, self.testDir) sys.path.insert(0, os.path.join(self.testDir, 'UnitTests')) # binDir = inspect.getsourcefile(ModuleLocator) # binDir = binDir.replace("__init__.py", "bin") # # if not binDir in os.environ['PATH']: # os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) open(os.path.join(self.testDir, 'UnitTests', '__init__.py'), 'w').close() shutil.copyfile( os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'), os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1', 'Report.pkl'))
Sample/Tester for a production workflow that writes several datasets and associated merges for each dataset. Production task produces three output datasets, Merge tasks are used to merge those datasets """ from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper # // # // Set up the basic workload task and step structure #// workload = newWorkload("MultiOutputs") workload.setOwner("WMTest") workload.setStartPolicy('MonteCarlo') workload.setEndPolicy('SingleShot') workload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) # // # // set up the production task #// production = workload.newTask("Production") #WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction" production.setTaskType("Merge") production.addProduction(totalEvents=1000) prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1")
#!/usr/bin/env python """ _Tier1ReRecoWorkload_ """ from WMCore.WMSpec.WMWorkload import newWorkload DBSURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" # // # // Set up the basic workload task and step structure #// workload = newWorkload("MultiTaskReReco") workload.setOwner("WMTest") workload.setStartPolicy('DatasetBlock') workload.setEndPolicy('SingleShot') # // # // set up the production task #// rereco = workload.newTask("ReReco1") rerecoCmssw = rereco.makeStep("cmsRun1") rerecoCmssw.setStepType("CMSSW") skimStageOut = rerecoCmssw.addStep("stageOut1") skimStageOut.setStepType("StageOut") skimLogArch = rerecoCmssw.addStep("logArch1") skimLogArch.setStepType("LogArchive") rereco.applyTemplates() rereco.setSplittingAlgorithm("FileBased", files_per_job = 1)
def setupExpressWorkflow(self): """ _setupExpressWorkflow_ Populate WMBS with a express-like workflow, every subscription must be unfinished at first """ workflowName = 'Express_Run481516_StreamZFast' secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT', 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM', 'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'] alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd' dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged' self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Harvesting', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) expressTask = workload.newTask('Express') for task in secondLevelTasks: secondLevelTask = expressTask.addTask(task) if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO': secondLevelTask.addTask(alcaHarvestTask) elif task == 'ExpressMergewrite_StreamZFast_DQM': secondLevelTask.addTask(dqmHarvestTask) specPath = os.path.join(self.testDir, 'Express.pkl') workload.save(specPath) # Populate WMBS sharedFileset = Fileset(name='TestFileset') sharedFileset.create() sharedFileset.markOpen(False) options = {'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0'} topLevelWorkflow = Workflow(task='/%s/Express' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(sharedFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topLevelSub) for task in [x for x in secondLevelTasks if not x.count('CleanupUnmerged')]: secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options) secondLevelWorkflow.create() mergeSub = Subscription(sharedFileset, secondLevelWorkflow) mergeSub.create() self.stateMap['Harvesting'].append(mergeSub) for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask), ('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]: harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child), **options) harvestingWorkflow.create() harvestingSub = Subscription(sharedFileset, harvestingWorkflow) harvestingSub.create() self.stateMap['Processing Done'].append(harvestingSub) return
Sample/Tester for a production workflow that writes several datasets and associated merges for each dataset. Production task produces three output datasets, Merge tasks are used to merge those datasets """ from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper # // # // Set up the basic workload task and step structure #// workload = newWorkload("MultiProduction") workload.setOwner("WMTest") workload.setStartPolicy('MonteCarlo') workload.setEndPolicy('SingleShot') # // # // set up the production task #// production = workload.newTask("Production") #WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction" production.setTaskType("Merge") production.addProduction(totalEvents = 1000) prodCmssw = production.makeStep("cmsRun1") prodCmssw.setStepType("CMSSW") prodStageOut = prodCmssw.addStep("stageOut1") prodStageOut.setStepType("StageOut")
_BasicProductionWorkload_ Sample/Tester for a simple production workflow and associated merge. Production task produces a single output dataset, Merge task is used to merge that dataset """ from WMCore.WMSpec.WMWorkload import newWorkload from WMCore.WMSpec.WMStep import makeWMStep from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper # // # // Set up the basic workload task and step structure #// workload = newWorkload("BasicProduction") workload.setOwner("WMTest") workload.setStartPolicy('MonteCarlo', **{ 'SliceType': 'NumberOfEvents', 'SliceSize': 100 }) workload.setEndPolicy('SingleShot') # // # // set up the production task #// production = workload.newTask("Production") production.addProduction(totalevents=1000) #WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction" production.setTaskType("Merge") production.setSplittingAlgorithm("EventBased", events_per_job=100) prodCmssw = production.makeStep("cmsRun1")
def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.reqDBWriter.insertGenericRequest(schema[0]) self.assertEquals(result[0]['ok'], True, 'insert fail') result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed") self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned") self.assertEquals(result,'Error: document not found') result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']}) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']}) self.assertEquals(result, 'Error: document not found') totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5} result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats) self.assertEquals(result, 'Error: document not found') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) properties = {"RequestPriority": spec1.priority(), 'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec1.listOutputDatasets()} result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties) self.assertEquals(result, 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") properties = {"RequestPriority": spec2.priority(), 'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec2.listOutputDatasets()} result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties) self.assertEquals(result, 'Error: document not found') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestByStatus(["failed"]) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName']) self.assertEquals(requests.keys(), [schema[0]['RequestName']])
def executeCMSSWStack(opts, scram): def getOutputModules(): pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\ "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\ "tweakJson = makeTweak(config.process).jsondictionary();"+\ "print tweakJson[\"process\"][\"outputModules_\"]" with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR): ret = scram("python -c '%s'" % pythonScript, runtimeDir=os.getcwd()) if ret > 0: msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic( ) handleException("FAILED", EC_CMSRunWrapper, msg) mintime() sys.exit(EC_CMSRunWrapper) output = literal_eval(scram.getStdout()) return output cmssw = CMSSW() cmssw.stepName = "cmsRun" cmssw.step = WMStep(cmssw.stepName) CMSSWTemplate().install(cmssw.step) cmssw.task = makeWMTask(cmssw.stepName) cmssw.workload = newWorkload(cmssw.stepName) cmssw.step.application.setup.softwareEnvironment = '' cmssw.step.application.setup.scramArch = opts.scramArch cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion cmssw.step.application.configuration.section_("arguments") cmssw.step.application.configuration.arguments.globalTag = "" for output in getOutputModules(): cmssw.step.output.modules.section_(output) getattr(cmssw.step.output.modules, output).primaryDataset = '' getattr(cmssw.step.output.modules, output).processedDataset = '' getattr(cmssw.step.output.modules, output).dataTier = '' #cmssw.step.application.command.arguments = '' #TODO cmssw.step.user.inputSandboxes = [opts.archiveJob] cmssw.step.user.userFiles = opts.userFiles or '' #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles. #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime.. cmssw.job = {'counter': 0, 'workflow': 'unused'} cmssw.step.user.lfnBase = '/store/temp/user/' cmssw.step.section_("builder") cmssw.step.builder.workingDir = os.getcwd() cmssw.step.runtime.invokeCommand = 'python' cmssw.step.runtime.scramPreDir = os.getcwd() cmssw.step.runtime.preScripts = [] cmssw.step.runtime.scramPreScripts = [ ('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' + '--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' + '--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' + '--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' + '--maxRuntime=%s') % (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis, opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun, opts.seeding, opts.lheInputFiles, opts.oneEventMode, opts.eventsPerLumi, opts.maxRuntime) ] cmssw.step.section_("execution") #exitStatus of cmsRun is set here cmssw.report = Report("cmsRun") #report is loaded and put here cmssw.execute() return cmssw
def makeWorkload(self, schema): workload = newWorkload(schema['RequestName']).data return workload