def setUp(self):
"""
_setUp_
"""
self.manager = SeederManager()
self.seedlistForRandom = {
"simMuonRPCDigis.initialSeed": None,
"simEcalUnsuppressedDigis.initialSeed": None,
"simCastorDigis.initialSeed": None,
"generator.initialSeed": None,
"simSiStripDigis.initialSeed": None,
"LHCTransport.initialSeed": None,
"mix.initialSeed": None,
"simHcalUnsuppressedDigis.initialSeed": None,
"theSource.initialSeed": None,
"simMuonCSCDigis.initialSeed": None,
"VtxSmeared.initialSeed": None,
"g4SimHits.initialSeed": None,
"simSiPixelDigis.initialSeed": None,
"simMuonDTDigis.initialSeed": None,
"evtgenproducer.initialSeed": None,
}
return
def __call__(self, parameters):
"""
Poller for looking in all active subscriptions for jobs that need to be made.
"""
logging.info("In JobCreatorWorker.__call__")
myThread = threading.currentThread()
for entry in parameters:
# This retrieves a single subscription
subscriptionID = entry.get('subscription')
if subscriptionID < 0:
logging.error("Got non-existant subscription")
logging.error("Assuming parameters in error: returning")
return subscriptionID
myThread.transaction.begin()
logging.info("About to call subscription %i" %subscriptionID)
wmbsSubscription = Subscription(id = subscriptionID)
wmbsSubscription.load()
wmbsSubscription["workflow"].load()
workflow = wmbsSubscription["workflow"]
wmWorkload = retrieveWMSpec(wmbsSubscription)
if not workflow.task or not wmWorkload:
# Then we have a problem
# We have no sandbox
# We NEED a sandbox
# Abort this subscription!
# But do NOT fail
# We have no way of marking a subscription as bad per se
# We'll have to just keep skipping it
wmTask = None
seederList = []
logging.error("Have no task for workflow %i" % (workflow.id))
logging.error("Aborting Subscription %i" % (subscriptionID))
continue
else:
wmTask = wmWorkload.getTaskByPath(workflow.task)
if hasattr(wmTask.data, 'seeders'):
manager = SeederManager(wmTask)
seederList = manager.getSeederList()
else:
seederList = []
logging.info("About to enter JobFactory")
logging.debug("Going to call wmbsJobFactory with limit %i" % (self.limit))
# My hope is that the job factory is smart enough only to split un-split jobs
wmbsJobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = wmbsSubscription,
generators=seederList,
limit = self.limit)
splitParams = retrieveJobSplitParams(wmWorkload, workflow.task)
logging.debug("Split Params: %s" % splitParams)
continueSubscription = True
myThread.transaction.commit()
# Turn on the jobFactory
myThread.transaction.begin()
wmbsJobFactory.open()
# Create a function to hold it
jobSplittingFunction = runSplitter(jobFactory = wmbsJobFactory,
splitParams = splitParams)
while continueSubscription:
# This loop runs over the jobFactory,
# using yield statements and a pre-existing proxy to
# generate and process new jobs
# First we need the jobs.
try:
wmbsJobGroups = jobSplittingFunction.next()
logging.info("Retrieved %i jobGroups from jobSplitter" % (len(wmbsJobGroups)))
except StopIteration:
# If you receive a stopIteration, we're done
logging.info("Completed iteration over subscription %i" % (subscriptionID))
continueSubscription = False
continue
# Now we get to find out what job they are.
countJobs = self.daoFactory(classname = "Jobs.GetNumberOfJobsPerWorkflow")
jobNumber = countJobs.execute(workflow = workflow.id,
conn = myThread.transaction.conn,
transaction = True)
logging.debug("Have %i jobs for this workflow already" % (jobNumber))
for wmbsJobGroup in wmbsJobGroups:
logging.debug("Processing jobGroup %i" % (wmbsJobGroup.exists()))
logging.debug("Processing %i jobs" % (len(wmbsJobGroup.jobs)) )
# Create a directory
self.createWorkArea.processJobs(jobGroup = wmbsJobGroup,
startDir = self.jobCacheDir,
workflow = workflow,
wmWorkload = wmWorkload,
transaction = myThread.transaction,
conn = myThread.transaction.conn)
for job in wmbsJobGroup.jobs:
jobNumber += 1
self.saveJob(job = job, workflow = workflow,
wmTask = wmTask, jobNumber = jobNumber)
self.advanceJobGroup(wmbsJobGroup)
logging.debug("Finished call for jobGroup %i" \
% (wmbsJobGroup.exists()))
# END: while loop over jobSplitter
myThread.transaction.commit()
# About to reset everything
wmbsJobGroups = None
wmTask = None
wmWorkload = None
splitParams = None
wmbsJobFactory = None
gc.collect()
# About to check memory
doMemoryCheck("About to get memory references: End of subscription loop")
# Final memory check
doMemoryCheck("About to get memory references: End of __call__()")
logging.debug("About to return from JobCreatorWorker.__call__()")
return parameters
def __call__(self, parameters):
"""
Poller for looking in all active subscriptions for jobs that need to be made.
"""
logging.info("In JobCreatorWorker.__call__")
myThread = threading.currentThread()
for entry in parameters:
# This retrieves a single subscription
subscriptionID = entry.get('subscription')
if subscriptionID < 0:
logging.error("Got non-existant subscription")
logging.error("Assuming parameters in error: returning")
return subscriptionID
myThread.transaction.begin()
logging.info("About to call subscription %i" %subscriptionID)
wmbsSubscription = Subscription(id = subscriptionID)
wmbsSubscription.load()
wmbsSubscription["workflow"].load()
workflow = wmbsSubscription["workflow"]
wmWorkload = retrieveWMSpec(wmbsSubscription)
if not workflow.task or not wmWorkload:
# Then we have a problem
# We have no sandbox
# We NEED a sandbox
# Abort this subscription!
# But do NOT fail
# We have no way of marking a subscription as bad per se
# We'll have to just keep skipping it
wmTask = None
seederList = []
logging.error("Have no task for workflow %i" % (workflow.id))
logging.error("Aborting Subscription %i" % (subscriptionID))
continue
else:
wmTask = wmWorkload.getTaskByPath(workflow.task)
if hasattr(wmTask.data, 'seeders'):
manager = SeederManager(wmTask)
seederList = manager.getSeederList()
else:
seederList = []
logging.info("About to enter JobFactory")
logging.debug("Going to call wmbsJobFactory with limit %i" % (self.limit))
# My hope is that the job factory is smart enough only to split un-split jobs
wmbsJobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = wmbsSubscription,
generators=seederList,
limit = self.limit)
splitParams = retrieveJobSplitParams(wmWorkload, workflow.task)
logging.debug("Split Params: %s" % splitParams)
continueSubscription = True
myThread.transaction.commit()
# Turn on the jobFactory
myThread.transaction.begin()
wmbsJobFactory.open()
# Create a function to hold it
jobSplittingFunction = runSplitter(jobFactory = wmbsJobFactory,
splitParams = splitParams)
while continueSubscription:
# This loop runs over the jobFactory,
# using yield statements and a pre-existing proxy to
# generate and process new jobs
# First we need the jobs.
try:
wmbsJobGroups = next(jobSplittingFunction)
logging.info("Retrieved %i jobGroups from jobSplitter" % (len(wmbsJobGroups)))
except StopIteration:
# If you receive a stopIteration, we're done
logging.info("Completed iteration over subscription %i" % (subscriptionID))
continueSubscription = False
continue
# Now we get to find out what job they are.
countJobs = self.daoFactory(classname = "Jobs.GetNumberOfJobsPerWorkflow")
jobNumber = countJobs.execute(workflow = workflow.id,
conn = myThread.transaction.conn,
transaction = True)
logging.debug("Have %i jobs for this workflow already" % (jobNumber))
for wmbsJobGroup in wmbsJobGroups:
logging.debug("Processing jobGroup %i" % (wmbsJobGroup.exists()))
logging.debug("Processing %i jobs" % (len(wmbsJobGroup.jobs)) )
# Create a directory
self.createWorkArea.processJobs(jobGroup = wmbsJobGroup,
startDir = self.jobCacheDir,
workflow = workflow,
wmWorkload = wmWorkload,
transaction = myThread.transaction,
conn = myThread.transaction.conn)
for job in wmbsJobGroup.jobs:
jobNumber += 1
self.saveJob(job = job, workflow = workflow,
wmTask = wmTask, jobNumber = jobNumber)
self.advanceJobGroup(wmbsJobGroup)
logging.debug("Finished call for jobGroup %i" \
% (wmbsJobGroup.exists()))
# END: while loop over jobSplitter
myThread.transaction.commit()
# About to reset everything
wmbsJobGroups = None
wmTask = None
wmWorkload = None
splitParams = None
wmbsJobFactory = None
gc.collect()
# About to check memory
doMemoryCheck("About to get memory references: End of subscription loop")
# Final memory check
doMemoryCheck("About to get memory references: End of __call__()")
logging.debug("About to return from JobCreatorWorker.__call__()")
return parameters
class SeederTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
"""
self.manager = SeederManager()
self.seedlistForRandom = {
"simMuonRPCDigis.initialSeed": None,
"simEcalUnsuppressedDigis.initialSeed": None,
"simCastorDigis.initialSeed": None,
"generator.initialSeed": None,
"simSiStripDigis.initialSeed": None,
"LHCTransport.initialSeed": None,
"mix.initialSeed": None,
"simHcalUnsuppressedDigis.initialSeed": None,
"theSource.initialSeed": None,
"simMuonCSCDigis.initialSeed": None,
"VtxSmeared.initialSeed": None,
"g4SimHits.initialSeed": None,
"simSiPixelDigis.initialSeed": None,
"simMuonDTDigis.initialSeed": None,
"evtgenproducer.initialSeed": None,
}
return
def tearDown(self):
"""
_tearDown_
"""
# Do nothing
return
def oneHundredFiles(self, splittingAlgo="EventBased", jobType="Processing"):
"""
_oneHundredFiles_
Generate a WMBS data stack representing 100 files for job splitter
testing
"""
fileset1 = Fileset(name="EventBasedFiles1")
for i in range(0, 100):
f = File(
"/store/MultipleFileSplit%s.root" % i, 1000, 100, 10 + i, 12312 # lfn # size # events # run # lumi
)
f["locations"].add("BULLSHIT")
fileset1.addFile(f)
work = Workflow()
subscription1 = Subscription(fileset=fileset1, workflow=work, split_algo=splittingAlgo, type=jobType)
splitter = SplitterFactory()
jobfactory = splitter(subscription1)
jobs = jobfactory(events_per_job=100)
# for jobGroup in jobs:
# yield jobGroup
self.manager.addSeeder("RandomSeeder", **self.seedlistForRandom)
self.manager.addSeeder("RunAndLumiSeeder")
return jobs
def testSimpleFiles(self):
"""
_testSimpleFiles_
test using one hundred files that we can save the attributes in a job
"""
jobs = self.oneHundredFiles()
for jobGrp in jobs:
self.manager(jobGrp)
for jobGrp in jobs:
count = 0
for job in jobGrp.jobs:
conf = job.getBaggage()
self.assertEqual(hasattr(conf.RandomSeeder.evtgenproducer, "initialSeed"), True)
self.assertEqual(hasattr(conf.RandomSeeder.generator, "initialSeed"), True)
self.assertEqual(job["mask"]["FirstLumi"], count % 11)
self.assertEqual(job["mask"]["FirstRun"], (count / 11) + 1)
count += 1
return
def testWMTask(self):
"""
_testWMTask_
Test whether or not we can read the seeder parameters out of a WMTask.
Also tests RandomSeeder and RunAndLumiSeeder
"""
task1 = makeWMTask("task1")
randomDict = {"generator.initialSeed": None, "evtgenproducer.initialSeed": None, "MAXINT": 10000}
lumiDict = {"lumi_per_run": 5}
task1.addGenerator("RandomSeeder", **randomDict)
task1.addGenerator("RunAndLumiSeeder", **lumiDict)
manager = SeederManager(task=task1)
jobs = self.oneHundredFiles()
for jobGrp in jobs:
manager(jobGrp)
for jobGrp in jobs:
count = 0
for job in jobGrp.jobs:
conf = job.getBaggage()
self.assertEqual(hasattr(conf.RandomSeeder.evtgenproducer, "initialSeed"), True)
self.assertEqual(hasattr(conf.RandomSeeder.generator, "initialSeed"), True)
self.assertEqual(job["mask"]["FirstLumi"], count % 6)
self.assertEqual(job["mask"]["FirstRun"], (count / 6) + 1)
x = conf.RandomSeeder.generator.initialSeed
assert x > 0, "ERROR: producing negative random numbers"
assert x < 10000, "ERROR: MAXINT tag failed; producing bad random number %i" % (x)
count += 1
return
def testPresetSeeder(self):
"""
_testPresetSeeder_
Test whether the PresetSeeder works
"""
task1 = makeWMTask("task2")
seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001}
task1.addGenerator("PresetSeeder", **seederDict)
manager = SeederManager(task=task1)
jobs = self.oneHundredFiles()
for jobGrp in jobs:
manager(jobGrp)
for jobGrp in jobs:
count = 0
for job in jobGrp.jobs:
conf = job.getBaggage()
self.assertEqual(conf.PresetSeeder.evtgenproducer.initialSeed, 1001)
self.assertEqual(conf.PresetSeeder.generator.initialSeed, 1001)
return
def pollSubscriptions(self):
"""
Poller for looking in all active subscriptions for jobs that need to be made.
"""
logging.info("Beginning JobCreator.pollSubscriptions() cycle.")
myThread = threading.currentThread()
#First, get list of Subscriptions
subscriptions = self.subscriptionList.execute()
# Okay, now we have a list of subscriptions
for subscriptionID in subscriptions:
wmbsSubscription = Subscription(id = subscriptionID)
try:
wmbsSubscription.load()
except IndexError:
# This happens when the subscription no longer exists
# i.e., someone executed a kill() function on the database
# while the JobCreator was in cycle
# Ignore this subscription
msg = "JobCreator cannot load subscription %i" % subscriptionID
logging.error(msg)
self.sendAlert(6, msg = msg)
continue
workflow = Workflow(id = wmbsSubscription["workflow"].id)
workflow.load()
wmbsSubscription['workflow'] = workflow
wmWorkload = retrieveWMSpec(workflow = workflow)
if not workflow.task or not wmWorkload:
# Then we have a problem
# We NEED a sandbox
# Abort this subscription!
# But do NOT fail
# We have no way of marking a subscription as bad per se
# We'll have to just keep skipping it
msg = "Have no task for workflow %i\n" % (workflow.id)
msg += "Aborting Subscription %i" % (subscriptionID)
logging.error(msg)
self.sendAlert(1, msg = msg)
continue
logging.debug("Have loaded subscription %i with workflow %i\n" % (subscriptionID, workflow.id))
# Set task object
wmTask = wmWorkload.getTaskByPath(workflow.task)
if hasattr(wmTask.data, 'seeders'):
manager = SeederManager(wmTask)
seederList = manager.getSeederList()
else:
seederList = []
logging.debug("Going to call wmbsJobFactory for sub %i with limit %i" % (subscriptionID, self.limit))
# My hope is that the job factory is smart enough only to split un-split jobs
wmbsJobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = wmbsSubscription,
generators=seederList,
limit = self.limit)
splitParams = retrieveJobSplitParams(wmWorkload, workflow.task)
logging.debug("Split Params: %s" % splitParams)
# Turn on the jobFactory
wmbsJobFactory.open()
# Create a function to hold it
jobSplittingFunction = runSplitter(jobFactory = wmbsJobFactory,
splitParams = splitParams)
# Now we get to find out how many jobs there are.
jobNumber = self.countJobs.execute(workflow = workflow.id,
conn = myThread.transaction.conn,
transaction = True)
jobNumber += splitParams.get('initial_lfn_counter', 0)
logging.debug("Have %i jobs for this workflow already" % (jobNumber))
continueSubscription = True
while continueSubscription:
# This loop runs over the jobFactory,
# using yield statements and a pre-existing proxy to
# generate and process new jobs
# First we need the jobs.
myThread.transaction.begin()
try:
wmbsJobGroups = jobSplittingFunction.next()
logging.info("Retrieved %i jobGroups from jobSplitter" % (len(wmbsJobGroups)))
except StopIteration:
# If you receive a stopIteration, we're done
logging.info("Completed iteration over subscription %i" % (subscriptionID))
continueSubscription = False
myThread.transaction.commit()
break
# If we have no jobGroups, we're done
if len(wmbsJobGroups) == 0:
logging.info("Found end in iteration over subscription %i" % (subscriptionID))
continueSubscription = False
myThread.transaction.commit()
break
# Assemble a dict of all the info
processDict = {'workflow': workflow,
'wmWorkload': wmWorkload, 'wmTaskName': wmTask.getPathName(),
'jobNumber': jobNumber, 'sandbox': wmTask.data.input.sandbox,
'wmTaskPrio': wmTask.getTaskPriority(),
'owner': wmWorkload.getOwner().get('name', None),
'ownerDN': wmWorkload.getOwner().get('dn', None)}
tempSubscription = Subscription(id = wmbsSubscription['id'])
nameDictList = []
for wmbsJobGroup in wmbsJobGroups:
# For each jobGroup, put a dictionary
# together and run it with creatorProcess
jobsInGroup = len(wmbsJobGroup.jobs)
wmbsJobGroup.subscription = tempSubscription
tempDict = {}
tempDict.update(processDict)
tempDict['jobGroup'] = wmbsJobGroup
tempDict['swVersion'] = wmTask.getSwVersion()
tempDict['scramArch'] = wmTask.getScramArch()
jobGroup = creatorProcess(work = tempDict,
jobCacheDir = self.jobCacheDir)
jobNumber += jobsInGroup
# Set jobCache for group
for job in jobGroup.jobs:
nameDictList.append({'jobid':job['id'],
'cacheDir':job['cache_dir']})
job["user"] = wmWorkload.getOwner()["name"]
job["group"] = wmWorkload.getOwner()["group"]
# Set the caches in the database
try:
if len(nameDictList) > 0:
self.setBulkCache.execute(jobDictList = nameDictList,
conn = myThread.transaction.conn,
transaction = True)
except WMException:
raise
except Exception, ex:
msg = "Unknown exception while setting the bulk cache:\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
logging.debug("Error while setting bulkCache with following values: %s\n" % nameDictList)
raise JobCreatorException(msg)
# Advance the jobGroup in changeState
for wmbsJobGroup in wmbsJobGroups:
self.advanceJobGroup(wmbsJobGroup = wmbsJobGroup)
# Now end the transaction so that everything is wrapped
# in a single rollback
myThread.transaction.commit()
# END: While loop over jobFactory
# Close the jobFactory
wmbsJobFactory.close()
