def testMCWithLHE(self): """ _testMCWithLHE_ Create a MonteCarlo workflow with a variation on the type of work done, this refers to the previous LHEStepZero where the input can be .lhe files and there is more than one lumi per job. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["LheInputFiles"] = "True" defaultArguments["EventsPerJob"] = 200 defaultArguments["EventsPerLumi"] = 50 factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') splitting = productionTask.jobSplittingParameters() self.assertEqual(splitting["events_per_job"], 200) self.assertEqual(splitting["events_per_lumi"], 50) self.assertEqual(splitting["lheInputFiles"], True) self.assertFalse(splitting["deterministicPileup"]) return
def testPileupFetcherOnMC(self): pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments() pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO" pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW" pileupMcArgs["CouchURL"] = os.environ["COUCHURL"] pileupMcArgs["CouchDBName"] = "pileupfetcher_t" pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs) # now that the workload was created and args validated, we can add this PileupConfig pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"]) # Since this is test of the fetcher - The loading from WMBS isn't # really necessary because the fetching happens before the workflow # is inserted into WMBS: feed the workload instance directly into fetcher: fetcher = PileupFetcher() creator = SandboxCreator() pathBase = "%s/%s" % (self.testDir, testWorkload.name()) for topLevelTask in testWorkload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): # this is how the call to PileupFetcher is happening # from the SandboxCreator test task = WMTask.WMTaskHelper(taskNode) taskPath = "%s/WMSandbox/%s" % (pathBase, task.name()) fetcher.setWorkingDirectory(taskPath) # create Sandbox for the fetcher ... creator._makePathonPackage(taskPath) fetcher(task) self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testInvalidKeys_initialize_clone(self): """ Test some undefined/assignment user arguments in initialize_clone """ # test with unknown key requestArgs = {'Multicore': 1, 'WRONG': 'Alan'} originalArgs = deepcopy(mcOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS) with self.assertRaises(WMSpecFactoryException): MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs) # test with assignment key requestArgs = {'Multicore': 1, 'TrustSitelists': False} originalArgs = deepcopy(mcOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS) with self.assertRaises(WMSpecFactoryException): MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs) # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception) requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}} originalArgs = deepcopy(taskChainOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS) with self.assertRaises(WMSpecFactoryException): TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs) # make sure unsupported keys are remove from both top and inner dicts requestArgs = {} originalArgs = deepcopy(taskChainOriginalArgs) originalArgs['BAD'] = 'KEY' originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here'] originalArgs['Task1']['Alan'] = False cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS) self.assertFalse('BAD' in cloneArgs) self.assertFalse('SiteWhitelist' in cloneArgs['Task1']) self.assertFalse('Alan' in cloneArgs['Task1'])
def testMonteCarlo(self): """ _testMonteCarlo_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testPileupFetcherOnMC(self): pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments() pileupMcArgs["PileupConfig"] = {"cosmics": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"], "minbias": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]} pileupMcArgs["CouchURL"] = os.environ["COUCHURL"] pileupMcArgs["CouchDBName"] = "pileupfetcher_t" pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs) # Since this is test of the fetcher - The loading from WMBS isn't # really necessary because the fetching happens before the workflow # is inserted into WMBS: feed the workload instance directly into fetcher: fetcher = PileupFetcher() creator = SandboxCreator() pathBase = "%s/%s" % (self.testDir, testWorkload.name()) for topLevelTask in testWorkload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): # this is how the call to PileupFetcher is happening # from the SandboxCreator test task = WMTask.WMTaskHelper(taskNode) taskPath = "%s/WMSandbox/%s" % (pathBase, task.name()) fetcher.setWorkingDirectory(taskPath) # create Sandbox for the fetcher ... creator._makePathonPackage(taskPath) fetcher(task) self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testPileupFetcherOnMC(self): pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments() pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO" pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW" pileupMcArgs["CouchURL"] = os.environ["COUCHURL"] pileupMcArgs["CouchDBName"] = "pileupfetcher_t" pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", pileupMcArgs) # now that the workload was created and args validated, we can add this PileupConfig pileupMcArgs["PileupConfig"] = parsePileupConfig( pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"]) # Since this is test of the fetcher - The loading from WMBS isn't # really necessary because the fetching happens before the workflow # is inserted into WMBS: feed the workload instance directly into fetcher: fetcher = PileupFetcher() creator = SandboxCreator() pathBase = "%s/%s" % (self.testDir, testWorkload.name()) for topLevelTask in testWorkload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): # this is how the call to PileupFetcher is happening # from the SandboxCreator test task = WMTask.WMTaskHelper(taskNode) taskPath = "%s/WMSandbox/%s" % (pathBase, task.name()) fetcher.setWorkingDirectory(taskPath) # create Sandbox for the fetcher ... creator._makePathonPackage(taskPath) fetcher(task) self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testMCWithPileup(self): """ _testMCWithPileup_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() # Add pileup inputs defaultArguments["MCPileup"] = COSMICS_PU defaultArguments["DataPileup"] = DATA_PU defaultArguments["DeterministicPileup"] = True factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper() pileupData = cmsRunStep.getPileup() self.assertEqual(pileupData.data.dataset, [DATA_PU]) self.assertEqual(pileupData.mc.dataset, [COSMICS_PU]) splitting = productionTask.jobSplittingParameters() self.assertTrue(splitting["deterministicPileup"]) return
def testPileupFetcherOnMC(self): pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments() pileupMcArgs["PileupConfig"] = { "cosmics": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ], "minbias": [ "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO" ] } pileupMcArgs["CouchURL"] = os.environ["COUCHURL"] pileupMcArgs["CouchDBName"] = "pileupfetcher_t" pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", pileupMcArgs) # Since this is test of the fetcher - The loading from WMBS isn't # really necessary because the fetching happens before the workflow # is inserted into WMBS: feed the workload instance directly into fetcher: fetcher = PileupFetcher() creator = SandboxCreator() pathBase = "%s/%s" % (self.testDir, testWorkload.name()) for topLevelTask in testWorkload.taskIterator(): for taskNode in topLevelTask.nodeIterator(): # this is how the call to PileupFetcher is happening # from the SandboxCreator test task = WMTask.WMTaskHelper(taskNode) taskPath = "%s/WMSandbox/%s" % (pathBase, task.name()) fetcher.setWorkingDirectory(taskPath) # create Sandbox for the fetcher ... creator._makePathonPackage(taskPath) fetcher(task) self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def validateSchema(self, schema): """ _validateSchema_ Check for required fields, and some skim facts """ MonteCarloWorkloadFactory.validateSchema(self, schema) #Now let's validate new stuff self.validateEventsPerLumi(schema)
def testMonteCarloExtension(self): """ _testMonteCarloExtension_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstLumi"] = 10001 defaultArguments["EventsPerJob"] = 100 defaultArguments["FirstEvent"] = 10001 #defaultArguments["FirstEvent"] = 10001 initial_lfn_counter = 100 # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath( '/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def testMemCoresSettings(self): """ _testMemCoresSettings_ Make sure the multicore and memory setings are properly propagated to all tasks and steps. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) # test default values taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], 2300.0) # now test case where args are provided defaultArguments["Multicore"] = 6 defaultArguments["Memory"] = 4600.0 defaultArguments["EventStreams"] = 3 testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) if step == 'cmsRun1': self.assertEqual(stepHelper.getNumberOfCores(), defaultArguments["Multicore"]) self.assertEqual(stepHelper.getNumberOfStreams(), defaultArguments["EventStreams"]) else: self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], defaultArguments["Memory"]) return
def __call__(self, workloadName, arguments): """ Create a workload instance for a LHE Step0 request Just pass it down to MonteCarlo """ #Override splitting arguments # Splitting arguments timePerEvent = int(arguments.get('TimePerEvent', 60)) filterEfficiency = float(arguments.get('FilterEfficiency', 1.0)) totalTime = int(arguments.get('TotalTime', 9 * 3600)) self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency) if arguments.get("LheInputFiles", False) == True \ or arguments.get("LheInputFiles", False) == "True": self.lheInputFiles = True # These are mostly place holders because the job splitting algo and # parameters will be updated after the workflow has been created. eventsPerJob = int(totalTime/timePerEvent/filterEfficiency) self.prodJobSplitAlgo = arguments.setdefault("ProdJobSplitAlgo", 'EventBased') self.prodJobSplitArgs = arguments.setdefault("ProdJobSplitArgs", {"events_per_job": eventsPerJob, "events_per_lumi": arguments['EventsPerLumi']}) self.prodJobSplitArgs.setdefault("lheInputFiles", self.lheInputFiles) mcWorkload = MonteCarloWorkloadFactory.__call__(self, workloadName, arguments) mcWorkload.setBlockCloseSettings(mcWorkload.getBlockCloseMaxWaitTime(), 5, 250000000, mcWorkload.getBlockCloseMaxSize()) return mcWorkload
def testMemCoresSettings(self): """ _testMemCoresSettings_ Make sure the multicore and memory setings are properly propagated to all tasks and steps. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) # test default values taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], 2300.0) # now test case where args are provided defaultArguments["Multicore"] = 6 defaultArguments["Memory"] = 4600.0 defaultArguments["EventStreams"] = 3 testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) taskObj = testWorkload.getTask('Production') for step in ('cmsRun1', 'stageOut1', 'logArch1'): stepHelper = taskObj.getStepHelper(step) if step == 'cmsRun1': self.assertEqual(stepHelper.getNumberOfCores(), defaultArguments["Multicore"]) self.assertEqual(stepHelper.getNumberOfStreams(), defaultArguments["EventStreams"]) else: self.assertEqual(stepHelper.getNumberOfCores(), 1) self.assertEqual(stepHelper.getNumberOfStreams(), 0) # then test Memory requirements perfParams = taskObj.jobSplittingParameters()['performance'] self.assertEqual(perfParams['memoryRequirement'], defaultArguments["Memory"]) return
def getMCArgs(): mcArgs = MonteCarloWorkloadFactory.getTestArguments() mcArgs.update({ "CouchURL": None, "CouchDBName": None, "ConfigCacheDoc" : None }) mcArgs.pop('ConfigCacheDoc') return mcArgs
def getMCArgs(): mcArgs = MonteCarloWorkloadFactory.getTestArguments() mcArgs.update({ "CouchURL": None, "CouchDBName": None, "ConfigCacheDoc": None }) mcArgs.pop('ConfigCacheDoc') return mcArgs
def testMonteCarloExtension(self): """ _testMonteCarloExtension_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstLumi"] = 10001 defaultArguments["EventsPerJob"] = 100 defaultArguments["FirstEvent"] = 10001 # defaultArguments["FirstEvent"] = 10001 initial_lfn_counter = ( 100 ) # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath("/TestWorkload/Production") productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath("/TestWorkload/Production/ProductionMerge%s" % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def testMonteCarlo(self): """ _testMonteCarlo_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testInvalidKeys_initialize_clone(self): """ Test some undefined/assignment user arguments in initialize_clone """ # test with unknown key requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False} originalArgs = deepcopy(mcOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS) with self.assertRaises(WMSpecFactoryException): MonteCarloWorkloadFactory().factoryWorkloadConstruction( cloneArgs["RequestName"], cloneArgs) # test with assignment key requestArgs = { 'Multicore': 1, 'TrustSitelists': False, 'ReqMgr2Only': False } originalArgs = deepcopy(mcOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS) with self.assertRaises(WMSpecFactoryException): MonteCarloWorkloadFactory().factoryWorkloadConstruction( cloneArgs["RequestName"], cloneArgs) # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception) requestArgs = { 'SubRequestType': "RelVal", "Step2": { "LumisPerJob": 5, "Campaign": "Step2Camp" } } originalArgs = deepcopy(taskChainOriginalArgs) cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS) with self.assertRaises(WMSpecFactoryException): TaskChainWorkloadFactory().factoryWorkloadConstruction( cloneArgs["RequestName"], cloneArgs)
def __call__(self, workloadName, arguments): """ Create a workload instance for a LHE Step0 request Just pass it down to MonteCarlo """ #Override splitting arguments # Splitting arguments timePerEvent = int(arguments.get('TimePerEvent', 60)) filterEfficiency = float(arguments.get('FilterEfficiency', 1.0)) totalTime = int(arguments.get('TotalTime', 9 * 3600)) self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency) # These are mostly place holders because the job splitting algo and # parameters will be updated after the workflow has been created. eventsPerJob = int(totalTime/timePerEvent/filterEfficiency) self.prodJobSplitAlgo = arguments.setdefault("ProdJobSplitAlgo", 'EventBased') self.prodJobSplitArgs = arguments.setdefault("ProdJobSplitArgs", {"events_per_job": eventsPerJob, "events_per_lumi": arguments['EventsPerLumi']}) return MonteCarloWorkloadFactory.__call__(self, workloadName, arguments)
def __call__(self, workflowName, args): workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args) delattr(workload.taskIterator().next().steps().data.application.configuration, 'configCacheUrl') return workload
if len(sys.argv) != 3: print("Usage:") print("./injectMonteCarloWorkflow.py PROCESSING_VERSION NUM_EVENTS") sys.exit(1) else: arguments["ProcessingVersion"] = sys.argv[1] numEvents = int(sys.argv[2]) connectToDB() workloadName = "MonteCarlo-%s" % arguments["ProcessingVersion"] workloadFile = "MonteCarlo-%s.pkl" % arguments["ProcessingVersion"] os.mkdir(workloadName) factory = MonteCarloWorkloadFactory() workload = factory(workloadName, arguments) workloadPath = os.path.join(workloadName, workloadFile) workload.setOwner("*****@*****.**") workload.setSpecUrl(workloadPath) # Build a sandbox using TaskMaker taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName)) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadPath) myThread = threading.currentThread() myThread.transaction.begin() for workloadTask in workload.taskIterator():
def __init__(self): MonteCarloWorkloadFactory.__init__(self)
def makeWorkload(self, schema): factory = MonteCarloWorkloadFactory() return factory(schema['RequestName'], schema).data
Unittests for IteratorTools functions """ from __future__ import division, print_function from pprint import pprint import unittest from copy import deepcopy from WMCore.ReqMgr.DataStructs.Request import initialize_clone from WMCore.WMSpec.StdSpecs.MonteCarlo import MonteCarloWorkloadFactory from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException ### Spec arguments definition with only key and its default value MC_ARGS = MonteCarloWorkloadFactory.getWorkloadCreateArgs() RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs() STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs() TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs() # inner Task/Step definition STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs() TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs() ### Some original request dicts (ones to be cloned from) mcOriginalArgs = { 'Memory': 1234, 'TimePerEvent': 1.2, 'RequestType': 'MonteCarlo', "LheInputFiles": True, "ScramArch": ["slc6_amd64_gcc481"], "RequestName": "test_mc"
def __call__(self, workflowName, args): workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args) #delattr(workload.taskIterator().next().steps().data.application.configuration, # 'configCacheUrl') return workload
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Production', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputA'] expWfTasks = ['/TestWorkload/Production', '/TestWorkload/Production/LogCollect', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', '/TestWorkload/Production/ProductionMergeOutputA', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect'] expFsets = ['FILESET_DEFINED_DURING_RUNTIME', '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO', '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER', '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/unmerged-OutputARECO'] subMaps = ['FILESET_DEFINED_DURING_RUNTIME', (6, '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', 'MinFileBased', 'LogCollect'), (3, '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect', 'MinFileBased', 'LogCollect'), (8, '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/LogCollect', 'MinFileBased', 'LogCollect'), (7, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', 'SiblingProcessingBased', 'Cleanup'), (5, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionMergeOutputA', 'ParentlessMergeBySize', 'Merge'), (4, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', 'SiblingProcessingBased', 'Cleanup'), (2, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputB', 'ParentlessMergeBySize', 'Merge')] testArguments = MonteCarloWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = TEST_DB_NAME testArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets[0] = topFilesetName # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production') subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps) ### create another top level subscription myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets.append(topFilesetName) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def __init__(self): MonteCarloWorkloadFactory.__init__(self) self.lheInputFiles = False