def testInvalidSpecs(self): """Specs with no work""" mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs) # 0 events getFirstTask(mcspec).addProduction(totalEvents = 0) for task in mcspec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task) # -ve split size mcspec2 = monteCarloWorkload('testProdInvalid', mcArgs) mcspec2.data.policies.start.SliceSize = -100 for task in mcspec2.taskIterator(): self.assertRaises(WorkQueueWMSpecError, MonteCarlo(), mcspec2, task)
def testInvalidSpecs(self): """Specs with no work""" mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs) # 0 events getFirstTask(mcspec).addProduction(totalevents = 0) for task in mcspec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task) # -ve split size mcspec2 = monteCarloWorkload('testProdInvalid', mcArgs) mcspec2.data.policies.start.SliceSize = -100 for task in mcspec2.taskIterator(): self.assertRaises(WorkQueueWMSpecError, MonteCarlo(), mcspec2, task)
def testInvalidSpecs(self): """Specs with no work""" mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs) # 0 events getFirstTask(mcspec).addProduction(totalevents = 0) for task in mcspec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task)
def testBasicProductionWorkload(self): """Basic Production Workload""" # change split defaults for this test totalevents = 1000000 splitArgs = dict(SliceType = 'NumEvents', SliceSize = 100, MaxJobsPerElement = 5) BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(BasicProductionWorkload).addProduction(totalevents = totalevents) getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) for task in BasicProductionWorkload.taskIterator(): units = MonteCarlo(**splitArgs)(BasicProductionWorkload, task) SliceSize = BasicProductionWorkload.startPolicyParameters()['SliceSize'] self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) first_event = 1 first_lumi = 1 first_run = 1 for unit in units: self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement']) self.assertEqual(unit['WMSpec'], BasicProductionWorkload) self.assertEqual(unit['Task'], task) self.assertEqual(unit['Mask']['FirstEvent'], first_event) self.assertEqual(unit['Mask']['FirstLumi'], first_lumi) last_event = first_event + (SliceSize * unit['Jobs']) - 1 if last_event > totalevents: # this should be the last unit of work last_event = totalevents self.assertEqual(unit['Mask']['LastEvent'], last_event) self.assertEqual(unit['Mask']['LastLumi'], first_lumi + unit['Jobs'] - 1) self.assertEqual(unit['Mask']['FirstRun'], first_run) first_event = last_event + 1 first_lumi += unit['Jobs'] # one lumi per job self.assertEqual(unit['Mask']['LastEvent'], totalevents)
def testBasicProductionWorkload(self): """Basic Production Workload""" # change split defaults for this test totalevents = 1000000 splitArgs = dict(SliceType = 'NumEvents', SliceSize = 100, MaxJobsPerElement = 5) BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(BasicProductionWorkload).addProduction(totalevents = totalevents) getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) for task in BasicProductionWorkload.taskIterator(): units = MonteCarlo(**splitArgs)(BasicProductionWorkload, task) self.assertEqual(int(totalevents / (splitArgs['SliceSize'] * splitArgs['MaxJobsPerElement'])), len(units)) first_event = 1 first_lumi = 1 first_run = 1 for unit in units: self.assertEqual(int(splitArgs['MaxJobsPerElement']), unit['Jobs']) self.assertEqual(unit['WMSpec'], BasicProductionWorkload) self.assertEqual(unit['Task'], task) self.assertEqual(unit['Mask']['FirstEvent'], first_event) self.assertEqual(unit['Mask']['FirstLumi'], first_lumi) last_event = first_event + (self.splitArgs['SliceSize'] * unit['Jobs']) - 1 self.assertEqual(unit['Mask']['LastEvent'], last_event) self.assertEqual(unit['Mask']['LastLumi'], first_lumi + unit['Jobs'] - 1) self.assertEqual(unit['Mask']['FirstRun'], first_run) first_event = last_event + 1 first_lumi += unit['Jobs'] # one lumi per job self.assertEqual(last_event, totalevents)
def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"] mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"] mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec
def testExtremeSplits(self): """ _testExtremeSplits_ Make sure that the protection to avoid going over 2^32 works """ totalevents = 2**34 splitArgs = dict(SliceType='NumEvents', SliceSize=2**30, MaxJobsPerElement=7) LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', { 'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SliceSize'] }) getFirstTask(LHEProductionWorkload).setSiteWhitelist( ['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(LHEProductionWorkload).addProduction( totalEvents=totalevents) getFirstTask(LHEProductionWorkload).setSiteWhitelist( ['T2_XX_SiteA', 'T2_XX_SiteB']) for task in LHEProductionWorkload.taskIterator(): units, _, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task) SliceSize = LHEProductionWorkload.startPolicyParameters( )['SliceSize'] self.assertEqual( math.ceil(totalevents / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) self.assertEqual(len(units), 3, "Should produce 3 units") unit1 = units[0] unit2 = units[1] unit3 = units[2] self.assertEqual(unit1['Jobs'], 7, 'First unit produced more jobs than expected') self.assertEqual(unit1['Mask']['FirstEvent'], 1, 'First unit has a wrong first event') self.assertEqual(unit1['Mask']['LastEvent'], 7 * (2**30), 'First unit has a wrong last event') self.assertEqual(unit2['Jobs'], 7, 'Second unit produced more jobs than expected') self.assertEqual(unit2['Mask']['FirstEvent'], 2**30 + 1, 'Second unit has a wrong first event') self.assertEqual(unit2['Mask']['LastEvent'], 8 * (2**30), 'Second unit has a wrong last event') self.assertEqual(unit3['Jobs'], 2, 'Third unit produced more jobs than expected') self.assertEqual(unit3['Mask']['FirstEvent'], 2 * (2**30) + 1, 'Third unit has a wrong first event') self.assertEqual(unit3['Mask']['LastEvent'], 4 * (2**30), 'First unit has a wrong last event')
def testShiftedStartSplitting(self): """ _testShiftedStartSplitting_ Make sure that splitting by event plus events in per lumi works when the first event and lumi is not 1 """ totalevents = 542674 splitArgs = dict(SliceType = 'NumEvents', SliceSize = 47, MaxJobsPerElement = 5, SubSliceType = 'NumEventsPerLumi', SubSliceSize = 13) LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SubSliceSize']}) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(LHEProductionWorkload).addProduction(totalevents = totalevents) getFirstTask(LHEProductionWorkload).setFirstEventAndLumi(50,100) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) for task in LHEProductionWorkload.taskIterator(): units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task) SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize'] self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) first_event = 50 first_lumi = 100 first_run = 1 lumis_per_job = int(math.ceil(float(SliceSize) / splitArgs['SubSliceSize'])) for unit in units: self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement']) self.assertEqual(unit['WMSpec'], LHEProductionWorkload) self.assertEqual(unit['Task'], task) self.assertEqual(unit['Mask']['FirstEvent'], first_event) self.assertEqual(unit['Mask']['FirstLumi'], first_lumi) last_event = first_event + (SliceSize * unit['Jobs']) - 1 last_lumi = first_lumi + (lumis_per_job * unit['Jobs']) - 1 if last_event > totalevents: # this should be the last unit of work last_event = totalevents + 50 - 1 last_lumi = first_lumi last_lumi += math.ceil(((last_event - first_event + 1) % SliceSize) / splitArgs['SubSliceSize']) last_lumi += (lumis_per_job * (unit['Jobs'] - 1)) - 1 self.assertEqual(unit['Mask']['LastEvent'], last_event) self.assertEqual(unit['Mask']['LastLumi'], last_lumi) self.assertEqual(unit['Mask']['FirstRun'], first_run) first_event = last_event + 1 first_lumi = last_lumi + 1 self.assertEqual(unit['Mask']['LastEvent'], totalevents + 50 - 1)
def testShiftedStartSplitting(self): """ _testShiftedStartSplitting_ Make sure that splitting by event plus events in per lumi works when the first event and lumi is not 1 """ totalevents = 542674 splitArgs = dict(SliceType = 'NumEvents', SliceSize = 47, MaxJobsPerElement = 5, SubSliceType = 'NumEventsPerLumi', SubSliceSize = 13) LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SubSliceSize']}) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(LHEProductionWorkload).addProduction(totalEvents = totalevents) getFirstTask(LHEProductionWorkload).setFirstEventAndLumi(50,100) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) for task in LHEProductionWorkload.taskIterator(): units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task) SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize'] self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) first_event = 50 first_lumi = 100 first_run = 1 lumis_per_job = int(math.ceil(float(SliceSize) / splitArgs['SubSliceSize'])) for unit in units: self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement']) self.assertEqual(unit['WMSpec'], LHEProductionWorkload) self.assertEqual(unit['Task'], task) self.assertEqual(unit['Mask']['FirstEvent'], first_event) self.assertEqual(unit['Mask']['FirstLumi'], first_lumi) last_event = first_event + (SliceSize * unit['Jobs']) - 1 last_lumi = first_lumi + (lumis_per_job * unit['Jobs']) - 1 if last_event > totalevents: # this should be the last unit of work last_event = totalevents + 50 - 1 last_lumi = first_lumi last_lumi += math.ceil(((last_event - first_event + 1) % SliceSize) / splitArgs['SubSliceSize']) last_lumi += (lumis_per_job * (unit['Jobs'] - 1)) - 1 self.assertEqual(unit['Mask']['LastEvent'], last_event) self.assertEqual(unit['Mask']['LastLumi'], last_lumi) self.assertEqual(unit['Mask']['FirstRun'], first_run) first_event = last_event + 1 first_lumi = last_lumi + 1 self.assertEqual(unit['Mask']['LastEvent'], totalevents + 50 - 1)
def testExtremeSplits(self): """ _testExtremeSplits_ Make sure that the protection to avoid going over 2^32 works """ totalevents = 2**34 splitArgs = dict(SliceType = 'NumEvents', SliceSize = 2**30, MaxJobsPerElement = 7) LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SliceSize']}) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(LHEProductionWorkload).addProduction(totalevents = totalevents) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) for task in LHEProductionWorkload.taskIterator(): units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task) SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize'] self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) self.assertEqual(len(units), 3, "Should produce 3 units") unit1 = units[0] unit2 = units[1] unit3 = units[2] self.assertEqual(unit1['Jobs'], 7, 'First unit produced more jobs than expected') self.assertEqual(unit1['Mask']['FirstEvent'], 1, 'First unit has a wrong first event') self.assertEqual(unit1['Mask']['LastEvent'], 7*(2**30), 'First unit has a wrong last event') self.assertEqual(unit2['Jobs'], 7, 'Second unit produced more jobs than expected') self.assertEqual(unit2['Mask']['FirstEvent'], 2**30 + 1, 'Second unit has a wrong first event') self.assertEqual(unit2['Mask']['LastEvent'], 8*(2**30), 'Second unit has a wrong last event') self.assertEqual(unit3['Jobs'], 2, 'Third unit produced more jobs than expected') self.assertEqual(unit3['Mask']['FirstEvent'], 2*(2**30) + 1, 'Third unit has a wrong first event') self.assertEqual(unit3['Mask']['LastEvent'], 4*(2**30), 'First unit has a wrong last event')
def _getLHEProductionWorkload(self, splitArgs): totalevents = 1010 LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SubSliceSize']}) getFirstTask(LHEProductionWorkload).addProduction(totalEvents=totalevents) getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) return LHEProductionWorkload
def testBasicProductionWorkload(self): """Basic Production Workload""" # change split defaults for this test totalevents = 1000000 splitArgs = dict(SliceType='NumberOfEvents', SliceSize=100, MaxJobsPerElement=5) mcArgs["EventsPerJob"] = 100 BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) getFirstTask(BasicProductionWorkload).setSiteWhitelist( ['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(BasicProductionWorkload).addProduction( totalEvents=totalevents) getFirstTask(BasicProductionWorkload).setSiteWhitelist( ['T2_XX_SiteA', 'T2_XX_SiteB']) for task in BasicProductionWorkload.taskIterator(): units, _, _ = MonteCarlo(**splitArgs)(BasicProductionWorkload, task) SliceSize = BasicProductionWorkload.startPolicyParameters( )['SliceSize'] self.assertEqual( math.ceil(totalevents / (SliceSize * splitArgs['MaxJobsPerElement'])), len(units)) first_event = 1 first_lumi = 1 first_run = 1 for unit in units: self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement']) self.assertEqual(unit['WMSpec'], BasicProductionWorkload) self.assertEqual(unit['Task'], task) self.assertEqual(unit['Mask']['FirstEvent'], first_event) self.assertEqual(unit['Mask']['FirstLumi'], first_lumi) last_event = first_event + (SliceSize * unit['Jobs']) - 1 if last_event > totalevents: # this should be the last unit of work last_event = totalevents last_lumi = first_lumi + unit['Jobs'] - 1 self.assertEqual(unit['Mask']['LastEvent'], last_event) self.assertEqual(unit['Mask']['LastLumi'], last_lumi) self.assertEqual(unit['Mask']['FirstRun'], first_run) self.assertEqual(last_lumi - first_lumi + 1, unit['NumberOfLumis']) self.assertEqual(last_event - first_event + 1, unit['NumberOfEvents']) first_event = last_event + 1 first_lumi += unit['Jobs'] # one lumi per job self.assertEqual(unit['Mask']['LastEvent'], totalevents)
def _getLHEProductionWorkload(self, splitArgs): totalevents = 1010 LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs) LHEProductionWorkload.setJobSplittingParameters( getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', { 'events_per_job': splitArgs['SliceSize'], 'events_per_lumi': splitArgs['SubSliceSize'] }) getFirstTask(LHEProductionWorkload).addProduction( totalEvents=totalevents) getFirstTask(LHEProductionWorkload).setSiteWhitelist( ['T2_XX_SiteA', 'T2_XX_SiteB']) return LHEProductionWorkload
def setUp(self): """ If we dont have a wmspec file create one """ EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = False) #set up WMAgent config file for couchdb self.configFile = EmulatorSetup.setupWMAgentConfig() WorkQueueTestCase.setUp(self) # Basic production Spec self.spec = monteCarloWorkload('testProduction', mcArgs) getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB']) getFirstTask(self.spec).addProduction(totalevents = 10000) self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec')) self.spec.save(self.spec.specUrl()) # Sample Tier1 ReReco spec self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs) self.processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessing.spec')) self.processingSpec.save(self.processingSpec.specUrl()) # Sample Tier1 ReReco spec self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs) self.parentProcSpec.setSpecUrl(os.path.join(self.workDir, 'testParentProcessing.spec')) self.parentProcSpec.save(self.parentProcSpec.specUrl()) # ReReco spec with blacklist self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs) self.blacklistSpec.setSpecUrl(os.path.join(self.workDir, 'testBlacklist.spec')) getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA'] self.blacklistSpec.save(self.blacklistSpec.specUrl()) # ReReco spec with whitelist self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs) self.whitelistSpec.setSpecUrl(os.path.join(self.workDir, 'testWhitelist.spec')) getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB'] self.whitelistSpec.save(self.whitelistSpec.specUrl()) # setup Mock DBS and PhEDEx inputDataset = getFirstTask(self.processingSpec).inputDataset() self.dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) # Create queues globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB) self.globalQueue = globalQueue(DbName = self.globalQDB, InboxDbName = self.globalQInboxDB, QueueURL = globalCouchUrl) # self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue # PopulateFilesets = False, # ParentQueue = self.globalQueue, # CacheDir = None) # ignore mid queue as it causes database duplication's # copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration jobCouchConfig = Configuration() jobCouchConfig.section_("JobStateMachine") jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"] jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb" # copy bossAir configuration here since we don't want/need to pass whole configuration bossAirConfig = Configuration() bossAirConfig.section_("BossAir") bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins" bossAirConfig.BossAir.pluginNames = ["CondorPlugin"] bossAirConfig.section_("Agent") bossAirConfig.Agent.agentName = "TestAgent" self.localQueue = localQueue(DbName = self.localQDB, InboxDbName = self.localQInboxDB, ParentQueueCouchUrl = globalCouchUrl, JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, CacheDir = self.workDir) self.localQueue2 = localQueue(DbName = self.localQDB2, InboxDbName = self.localQInboxDB2, ParentQueueCouchUrl = globalCouchUrl, JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, CacheDir = self.workDir) # configuration for the Alerts messaging framework, work (alerts) and # control channel addresses to which alerts # these are destination addresses where AlertProcessor:Receiver listens config = Configuration() config.section_("Alert") config.Alert.address = "tcp://127.0.0.1:5557" config.Alert.controlAddr = "tcp://127.0.0.1:5559" # standalone queue for unit tests self.queue = WorkQueue(JobDumpConfig = jobCouchConfig, BossAirConfig = bossAirConfig, DbName = self.queueDB, InboxDbName = self.queueInboxDB, CacheDir = self.workDir, config = config) # create relevant sites in wmbs rc = ResourceControl() for site, se in self.queue.SiteDB.mapping.items(): rc.insertSite(site, 100, se, cmsName = site) daofactory = DAOFactory(package = "WMCore.WMBS", logger = threading.currentThread().logger, dbinterface = threading.currentThread().dbi) addLocation = daofactory(classname = "Locations.New") addLocation.execute(siteName = site, seName = se)
def createMCWMSpec(self, name='MonteCarloWorkload'): wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec
def testInvalidSpecs(self): """Complain on invalid WMSpecs""" # request != workflow name self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, self.processingSpec.specUrl(), request = 'fail_this') # invalid white list mcspec = monteCarloWorkload('testProductionInvalid', mcArgs) getFirstTask(mcspec).setSiteWhitelist('ThisIsInvalid') mcspec.setSpecUrl(os.path.join(self.workDir, 'testProductionInvalid.spec')) mcspec.save(mcspec.specUrl()) self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, mcspec.specUrl()) getFirstTask(mcspec).setSiteWhitelist([]) # 0 events getFirstTask(mcspec).addProduction(totalevents = 0) mcspec.save(mcspec.specUrl()) self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, mcspec.specUrl()) # no dataset processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs) processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessingInvalid.spec')) processingSpec.save(processingSpec.specUrl()) getFirstTask(processingSpec).data.input.dataset = None processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl()) # invalid dbs url processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs) processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessingInvalid.spec')) getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com' processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl()) # invalid dataset name processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs) processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessingInvalid.spec')) getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl()) # Cant have a slash in primary ds name - validation should fail getFirstTask(processingSpec).data.input.dataset.primary = 'a/b' processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl()) # dataset splitting with invalid run whitelist processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs) processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessingInvalid.spec')) processingSpec.setStartPolicy('Dataset') processingSpec.setRunWhitelist([666]) # not in this dataset processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl()) # block splitting with invalid run whitelist processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs) processingSpec.setSpecUrl(os.path.join(self.workDir, 'testProcessingInvalid.spec')) processingSpec.setStartPolicy('Block') processingSpec.setRunWhitelist([666]) # not in this dataset processingSpec.save(processingSpec.specUrl()) self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl())
def createMCWMSpec(self, name = 'MonteCarloWorkload'): wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents = 10000) return wmspec