def testInvalidSpecs(self):
"""Specs with no work"""
mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs)
# 0 events
getFirstTask(mcspec).addProduction(totalEvents = 0)
for task in mcspec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task)
# -ve split size
mcspec2 = monteCarloWorkload('testProdInvalid', mcArgs)
mcspec2.data.policies.start.SliceSize = -100
for task in mcspec2.taskIterator():
self.assertRaises(WorkQueueWMSpecError, MonteCarlo(), mcspec2, task)
def testInvalidSpecs(self):
"""Specs with no work"""
mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs)
# 0 events
getFirstTask(mcspec).addProduction(totalevents = 0)
for task in mcspec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task)
# -ve split size
mcspec2 = monteCarloWorkload('testProdInvalid', mcArgs)
mcspec2.data.policies.start.SliceSize = -100
for task in mcspec2.taskIterator():
self.assertRaises(WorkQueueWMSpecError, MonteCarlo(), mcspec2, task)
def testInvalidSpecs(self):
"""Specs with no work"""
mcspec = monteCarloWorkload('testProcessingInvalid', mcArgs)
# 0 events
getFirstTask(mcspec).addProduction(totalevents = 0)
for task in mcspec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, MonteCarlo(), mcspec, task)
def testBasicProductionWorkload(self):
"""Basic Production Workload"""
# change split defaults for this test
totalevents = 1000000
splitArgs = dict(SliceType = 'NumEvents', SliceSize = 100, MaxJobsPerElement = 5)
BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(BasicProductionWorkload).addProduction(totalevents = totalevents)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in BasicProductionWorkload.taskIterator():
units = MonteCarlo(**splitArgs)(BasicProductionWorkload, task)
SliceSize = BasicProductionWorkload.startPolicyParameters()['SliceSize']
self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
first_event = 1
first_lumi = 1
first_run = 1
for unit in units:
self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement'])
self.assertEqual(unit['WMSpec'], BasicProductionWorkload)
self.assertEqual(unit['Task'], task)
self.assertEqual(unit['Mask']['FirstEvent'], first_event)
self.assertEqual(unit['Mask']['FirstLumi'], first_lumi)
last_event = first_event + (SliceSize * unit['Jobs']) - 1
if last_event > totalevents:
# this should be the last unit of work
last_event = totalevents
self.assertEqual(unit['Mask']['LastEvent'], last_event)
self.assertEqual(unit['Mask']['LastLumi'], first_lumi + unit['Jobs'] - 1)
self.assertEqual(unit['Mask']['FirstRun'], first_run)
first_event = last_event + 1
first_lumi += unit['Jobs'] # one lumi per job
self.assertEqual(unit['Mask']['LastEvent'], totalevents)
def testBasicProductionWorkload(self):
"""Basic Production Workload"""
# change split defaults for this test
totalevents = 1000000
splitArgs = dict(SliceType = 'NumEvents', SliceSize = 100, MaxJobsPerElement = 5)
BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(BasicProductionWorkload).addProduction(totalevents = totalevents)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in BasicProductionWorkload.taskIterator():
units = MonteCarlo(**splitArgs)(BasicProductionWorkload, task)
self.assertEqual(int(totalevents / (splitArgs['SliceSize'] * splitArgs['MaxJobsPerElement'])),
len(units))
first_event = 1
first_lumi = 1
first_run = 1
for unit in units:
self.assertEqual(int(splitArgs['MaxJobsPerElement']), unit['Jobs'])
self.assertEqual(unit['WMSpec'], BasicProductionWorkload)
self.assertEqual(unit['Task'], task)
self.assertEqual(unit['Mask']['FirstEvent'], first_event)
self.assertEqual(unit['Mask']['FirstLumi'], first_lumi)
last_event = first_event + (self.splitArgs['SliceSize'] * unit['Jobs']) - 1
self.assertEqual(unit['Mask']['LastEvent'], last_event)
self.assertEqual(unit['Mask']['LastLumi'], first_lumi + unit['Jobs'] - 1)
self.assertEqual(unit['Mask']['FirstRun'], first_run)
first_event = last_event + 1
first_lumi += unit['Jobs'] # one lumi per job
self.assertEqual(last_event, totalevents)
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"]
mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"]
mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def testExtremeSplits(self):
"""
_testExtremeSplits_
Make sure that the protection to avoid going over 2^32 works
"""
totalevents = 2**34
splitArgs = dict(SliceType='NumEvents',
SliceSize=2**30,
MaxJobsPerElement=7)
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload',
mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {
'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SliceSize']
})
getFirstTask(LHEProductionWorkload).setSiteWhitelist(
['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(LHEProductionWorkload).addProduction(
totalEvents=totalevents)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(
['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in LHEProductionWorkload.taskIterator():
units, _, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task)
SliceSize = LHEProductionWorkload.startPolicyParameters(
)['SliceSize']
self.assertEqual(
math.ceil(totalevents /
(SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
self.assertEqual(len(units), 3, "Should produce 3 units")
unit1 = units[0]
unit2 = units[1]
unit3 = units[2]
self.assertEqual(unit1['Jobs'], 7,
'First unit produced more jobs than expected')
self.assertEqual(unit1['Mask']['FirstEvent'], 1,
'First unit has a wrong first event')
self.assertEqual(unit1['Mask']['LastEvent'], 7 * (2**30),
'First unit has a wrong last event')
self.assertEqual(unit2['Jobs'], 7,
'Second unit produced more jobs than expected')
self.assertEqual(unit2['Mask']['FirstEvent'], 2**30 + 1,
'Second unit has a wrong first event')
self.assertEqual(unit2['Mask']['LastEvent'], 8 * (2**30),
'Second unit has a wrong last event')
self.assertEqual(unit3['Jobs'], 2,
'Third unit produced more jobs than expected')
self.assertEqual(unit3['Mask']['FirstEvent'], 2 * (2**30) + 1,
'Third unit has a wrong first event')
self.assertEqual(unit3['Mask']['LastEvent'], 4 * (2**30),
'First unit has a wrong last event')
def testShiftedStartSplitting(self):
"""
_testShiftedStartSplitting_
Make sure that splitting by event plus events in per lumi works
when the first event and lumi is not 1
"""
totalevents = 542674
splitArgs = dict(SliceType = 'NumEvents', SliceSize = 47, MaxJobsPerElement = 5,
SubSliceType = 'NumEventsPerLumi', SubSliceSize = 13)
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(),
'EventBased',
{'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SubSliceSize']})
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(LHEProductionWorkload).addProduction(totalevents = totalevents)
getFirstTask(LHEProductionWorkload).setFirstEventAndLumi(50,100)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in LHEProductionWorkload.taskIterator():
units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task)
SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize']
self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
first_event = 50
first_lumi = 100
first_run = 1
lumis_per_job = int(math.ceil(float(SliceSize) /
splitArgs['SubSliceSize']))
for unit in units:
self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement'])
self.assertEqual(unit['WMSpec'], LHEProductionWorkload)
self.assertEqual(unit['Task'], task)
self.assertEqual(unit['Mask']['FirstEvent'], first_event)
self.assertEqual(unit['Mask']['FirstLumi'], first_lumi)
last_event = first_event + (SliceSize * unit['Jobs']) - 1
last_lumi = first_lumi + (lumis_per_job * unit['Jobs']) - 1
if last_event > totalevents:
# this should be the last unit of work
last_event = totalevents + 50 - 1
last_lumi = first_lumi
last_lumi += math.ceil(((last_event - first_event + 1) %
SliceSize) / splitArgs['SubSliceSize'])
last_lumi += (lumis_per_job * (unit['Jobs'] - 1)) - 1
self.assertEqual(unit['Mask']['LastEvent'], last_event)
self.assertEqual(unit['Mask']['LastLumi'], last_lumi)
self.assertEqual(unit['Mask']['FirstRun'], first_run)
first_event = last_event + 1
first_lumi = last_lumi + 1
self.assertEqual(unit['Mask']['LastEvent'], totalevents + 50 - 1)
def testShiftedStartSplitting(self):
"""
_testShiftedStartSplitting_
Make sure that splitting by event plus events in per lumi works
when the first event and lumi is not 1
"""
totalevents = 542674
splitArgs = dict(SliceType = 'NumEvents', SliceSize = 47, MaxJobsPerElement = 5,
SubSliceType = 'NumEventsPerLumi', SubSliceSize = 13)
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(),
'EventBased',
{'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SubSliceSize']})
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(LHEProductionWorkload).addProduction(totalEvents = totalevents)
getFirstTask(LHEProductionWorkload).setFirstEventAndLumi(50,100)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in LHEProductionWorkload.taskIterator():
units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task)
SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize']
self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
first_event = 50
first_lumi = 100
first_run = 1
lumis_per_job = int(math.ceil(float(SliceSize) /
splitArgs['SubSliceSize']))
for unit in units:
self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement'])
self.assertEqual(unit['WMSpec'], LHEProductionWorkload)
self.assertEqual(unit['Task'], task)
self.assertEqual(unit['Mask']['FirstEvent'], first_event)
self.assertEqual(unit['Mask']['FirstLumi'], first_lumi)
last_event = first_event + (SliceSize * unit['Jobs']) - 1
last_lumi = first_lumi + (lumis_per_job * unit['Jobs']) - 1
if last_event > totalevents:
# this should be the last unit of work
last_event = totalevents + 50 - 1
last_lumi = first_lumi
last_lumi += math.ceil(((last_event - first_event + 1) %
SliceSize) / splitArgs['SubSliceSize'])
last_lumi += (lumis_per_job * (unit['Jobs'] - 1)) - 1
self.assertEqual(unit['Mask']['LastEvent'], last_event)
self.assertEqual(unit['Mask']['LastLumi'], last_lumi)
self.assertEqual(unit['Mask']['FirstRun'], first_run)
first_event = last_event + 1
first_lumi = last_lumi + 1
self.assertEqual(unit['Mask']['LastEvent'], totalevents + 50 - 1)
def testExtremeSplits(self):
"""
_testExtremeSplits_
Make sure that the protection to avoid going over 2^32 works
"""
totalevents = 2**34
splitArgs = dict(SliceType = 'NumEvents', SliceSize = 2**30,
MaxJobsPerElement = 7)
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(),
'EventBased',
{'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SliceSize']})
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(LHEProductionWorkload).addProduction(totalevents = totalevents)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in LHEProductionWorkload.taskIterator():
units, _ = MonteCarlo(**splitArgs)(LHEProductionWorkload, task)
SliceSize = LHEProductionWorkload.startPolicyParameters()['SliceSize']
self.assertEqual(math.ceil(float(totalevents) / (SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
self.assertEqual(len(units), 3, "Should produce 3 units")
unit1 = units[0]
unit2 = units[1]
unit3 = units[2]
self.assertEqual(unit1['Jobs'],
7, 'First unit produced more jobs than expected')
self.assertEqual(unit1['Mask']['FirstEvent'],
1, 'First unit has a wrong first event')
self.assertEqual(unit1['Mask']['LastEvent'],
7*(2**30), 'First unit has a wrong last event')
self.assertEqual(unit2['Jobs'],
7, 'Second unit produced more jobs than expected')
self.assertEqual(unit2['Mask']['FirstEvent'],
2**30 + 1, 'Second unit has a wrong first event')
self.assertEqual(unit2['Mask']['LastEvent'],
8*(2**30), 'Second unit has a wrong last event')
self.assertEqual(unit3['Jobs'],
2, 'Third unit produced more jobs than expected')
self.assertEqual(unit3['Mask']['FirstEvent'],
2*(2**30) + 1, 'Third unit has a wrong first event')
self.assertEqual(unit3['Mask']['LastEvent'],
4*(2**30), 'First unit has a wrong last event')
def _getLHEProductionWorkload(self, splitArgs):
totalevents = 1010
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload', mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(),
'EventBased',
{'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SubSliceSize']})
getFirstTask(LHEProductionWorkload).addProduction(totalEvents=totalevents)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
return LHEProductionWorkload
def testBasicProductionWorkload(self):
"""Basic Production Workload"""
# change split defaults for this test
totalevents = 1000000
splitArgs = dict(SliceType='NumberOfEvents',
SliceSize=100,
MaxJobsPerElement=5)
mcArgs["EventsPerJob"] = 100
BasicProductionWorkload = monteCarloWorkload('MonteCarloWorkload',
mcArgs)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(
['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(BasicProductionWorkload).addProduction(
totalEvents=totalevents)
getFirstTask(BasicProductionWorkload).setSiteWhitelist(
['T2_XX_SiteA', 'T2_XX_SiteB'])
for task in BasicProductionWorkload.taskIterator():
units, _, _ = MonteCarlo(**splitArgs)(BasicProductionWorkload,
task)
SliceSize = BasicProductionWorkload.startPolicyParameters(
)['SliceSize']
self.assertEqual(
math.ceil(totalevents /
(SliceSize * splitArgs['MaxJobsPerElement'])),
len(units))
first_event = 1
first_lumi = 1
first_run = 1
for unit in units:
self.assertTrue(unit['Jobs'] <= splitArgs['MaxJobsPerElement'])
self.assertEqual(unit['WMSpec'], BasicProductionWorkload)
self.assertEqual(unit['Task'], task)
self.assertEqual(unit['Mask']['FirstEvent'], first_event)
self.assertEqual(unit['Mask']['FirstLumi'], first_lumi)
last_event = first_event + (SliceSize * unit['Jobs']) - 1
if last_event > totalevents:
# this should be the last unit of work
last_event = totalevents
last_lumi = first_lumi + unit['Jobs'] - 1
self.assertEqual(unit['Mask']['LastEvent'], last_event)
self.assertEqual(unit['Mask']['LastLumi'], last_lumi)
self.assertEqual(unit['Mask']['FirstRun'], first_run)
self.assertEqual(last_lumi - first_lumi + 1,
unit['NumberOfLumis'])
self.assertEqual(last_event - first_event + 1,
unit['NumberOfEvents'])
first_event = last_event + 1
first_lumi += unit['Jobs'] # one lumi per job
self.assertEqual(unit['Mask']['LastEvent'], totalevents)
def _getLHEProductionWorkload(self, splitArgs):
totalevents = 1010
LHEProductionWorkload = monteCarloWorkload('MonteCarloWorkload',
mcArgs)
LHEProductionWorkload.setJobSplittingParameters(
getFirstTask(LHEProductionWorkload).getPathName(), 'EventBased', {
'events_per_job': splitArgs['SliceSize'],
'events_per_lumi': splitArgs['SubSliceSize']
})
getFirstTask(LHEProductionWorkload).addProduction(
totalEvents=totalevents)
getFirstTask(LHEProductionWorkload).setSiteWhitelist(
['T2_XX_SiteA', 'T2_XX_SiteB'])
return LHEProductionWorkload
def setUp(self):
"""
If we dont have a wmspec file create one
"""
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
#set up WMAgent config file for couchdb
self.configFile = EmulatorSetup.setupWMAgentConfig()
WorkQueueTestCase.setUp(self)
# Basic production Spec
self.spec = monteCarloWorkload('testProduction', mcArgs)
getFirstTask(self.spec).setSiteWhitelist(['T2_XX_SiteA', 'T2_XX_SiteB'])
getFirstTask(self.spec).addProduction(totalevents = 10000)
self.spec.setSpecUrl(os.path.join(self.workDir, 'testworkflow.spec'))
self.spec.save(self.spec.specUrl())
# Sample Tier1 ReReco spec
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
self.processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessing.spec'))
self.processingSpec.save(self.processingSpec.specUrl())
# Sample Tier1 ReReco spec
self.parentProcSpec = rerecoWorkload('testParentProcessing', parentProcArgs)
self.parentProcSpec.setSpecUrl(os.path.join(self.workDir,
'testParentProcessing.spec'))
self.parentProcSpec.save(self.parentProcSpec.specUrl())
# ReReco spec with blacklist
self.blacklistSpec = rerecoWorkload('blacklistSpec', rerecoArgs)
self.blacklistSpec.setSpecUrl(os.path.join(self.workDir,
'testBlacklist.spec'))
getFirstTask(self.blacklistSpec).data.constraints.sites.blacklist = ['T2_XX_SiteA']
self.blacklistSpec.save(self.blacklistSpec.specUrl())
# ReReco spec with whitelist
self.whitelistSpec = rerecoWorkload('whitelistlistSpec', rerecoArgs)
self.whitelistSpec.setSpecUrl(os.path.join(self.workDir,
'testWhitelist.spec'))
getFirstTask(self.whitelistSpec).data.constraints.sites.whitelist = ['T2_XX_SiteB']
self.whitelistSpec.save(self.whitelistSpec.specUrl())
# setup Mock DBS and PhEDEx
inputDataset = getFirstTask(self.processingSpec).inputDataset()
self.dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
# Create queues
globalCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.globalQueue = globalQueue(DbName = self.globalQDB,
InboxDbName = self.globalQInboxDB,
QueueURL = globalCouchUrl)
# self.midQueue = WorkQueue(SplitByBlock = False, # mid-level queue
# PopulateFilesets = False,
# ParentQueue = self.globalQueue,
# CacheDir = None)
# ignore mid queue as it causes database duplication's
# copy jobStateMachine couchDB configuration here since we don't want/need to pass whole configuration
jobCouchConfig = Configuration()
jobCouchConfig.section_("JobStateMachine")
jobCouchConfig.JobStateMachine.couchurl = os.environ["COUCHURL"]
jobCouchConfig.JobStateMachine.couchDBName = "testcouchdb"
# copy bossAir configuration here since we don't want/need to pass whole configuration
bossAirConfig = Configuration()
bossAirConfig.section_("BossAir")
bossAirConfig.BossAir.pluginDir = "WMCore.BossAir.Plugins"
bossAirConfig.BossAir.pluginNames = ["CondorPlugin"]
bossAirConfig.section_("Agent")
bossAirConfig.Agent.agentName = "TestAgent"
self.localQueue = localQueue(DbName = self.localQDB,
InboxDbName = self.localQInboxDB,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
self.localQueue2 = localQueue(DbName = self.localQDB2,
InboxDbName = self.localQInboxDB2,
ParentQueueCouchUrl = globalCouchUrl,
JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
CacheDir = self.workDir)
# configuration for the Alerts messaging framework, work (alerts) and
# control channel addresses to which alerts
# these are destination addresses where AlertProcessor:Receiver listens
config = Configuration()
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# standalone queue for unit tests
self.queue = WorkQueue(JobDumpConfig = jobCouchConfig,
BossAirConfig = bossAirConfig,
DbName = self.queueDB,
InboxDbName = self.queueInboxDB,
CacheDir = self.workDir,
config = config)
# create relevant sites in wmbs
rc = ResourceControl()
for site, se in self.queue.SiteDB.mapping.items():
rc.insertSite(site, 100, se, cmsName = site)
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = threading.currentThread().logger,
dbinterface = threading.currentThread().dbi)
addLocation = daofactory(classname = "Locations.New")
addLocation.execute(siteName = site, seName = se)
def createMCWMSpec(self, name='MonteCarloWorkload'):
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def testInvalidSpecs(self):
"""Complain on invalid WMSpecs"""
# request != workflow name
self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork,
self.processingSpec.specUrl(),
request = 'fail_this')
# invalid white list
mcspec = monteCarloWorkload('testProductionInvalid', mcArgs)
getFirstTask(mcspec).setSiteWhitelist('ThisIsInvalid')
mcspec.setSpecUrl(os.path.join(self.workDir, 'testProductionInvalid.spec'))
mcspec.save(mcspec.specUrl())
self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, mcspec.specUrl())
getFirstTask(mcspec).setSiteWhitelist([])
# 0 events
getFirstTask(mcspec).addProduction(totalevents = 0)
mcspec.save(mcspec.specUrl())
self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, mcspec.specUrl())
# no dataset
processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs)
processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessingInvalid.spec'))
processingSpec.save(processingSpec.specUrl())
getFirstTask(processingSpec).data.input.dataset = None
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl())
# invalid dbs url
processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs)
processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessingInvalid.spec'))
getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl())
# invalid dataset name
processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs)
processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessingInvalid.spec'))
getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl())
# Cant have a slash in primary ds name - validation should fail
getFirstTask(processingSpec).data.input.dataset.primary = 'a/b'
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueWMSpecError, self.queue.queueWork, processingSpec.specUrl())
# dataset splitting with invalid run whitelist
processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs)
processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessingInvalid.spec'))
processingSpec.setStartPolicy('Dataset')
processingSpec.setRunWhitelist([666]) # not in this dataset
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl())
# block splitting with invalid run whitelist
processingSpec = rerecoWorkload('testProcessingInvalid', rerecoArgs)
processingSpec.setSpecUrl(os.path.join(self.workDir,
'testProcessingInvalid.spec'))
processingSpec.setStartPolicy('Block')
processingSpec.setRunWhitelist([666]) # not in this dataset
processingSpec.save(processingSpec.specUrl())
self.assertRaises(WorkQueueNoWorkError, self.queue.queueWork, processingSpec.specUrl())
def createMCWMSpec(self, name = 'MonteCarloWorkload'):
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents = 10000)
return wmspec
