tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
prodSourceLabel,
cloudName,nFiles=10,simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
tmpStat,pandaJobs,datasetToRegister,oldPandaIDs = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True)
if taskSpec.useEventService():
pandaJobs = gen.increaseEventServiceConsumers(pandaJobs,taskSpec.getNumEventServiceConsumer())
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
jobBroker.setTestMode(taskSpec.vo,taskSpec.prodSourceLabel)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None,
None,None,None,brokerageLockIDs)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
jobBroker.setLockID(taskSpec.vo,taskSpec.prodSourceLabel,123,0)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,taskSpec.prodSourceLabel)
if brokerageLockID != None:
brokerageLockIDs.append(brokerageLockID)
for brokeragelockID in brokerageLockIDs:
tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,taskSpec.prodSourceLabel,workQueue.queue_id,
brokeragelockID,True)
tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
tmpStat,pandaJobs,datasetToRegister,oldPandaIDs,parallelOutMap,outDsMap = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True,
splitter=splitter)
splitter = JobSplitter()
gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
False, taskSetupper, None, None, None, None,
brokerageLockIDs, False)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
tmpLog)
jobBroker.setLockID(taskSpec.vo, taskSpec.prodSourceLabel, 123, 0)
tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
inputChunk, taskParamMap)
brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,
taskSpec.prodSourceLabel)
if brokerageLockID != None:
brokerageLockIDs.append(brokerageLockID)
for brokeragelockID in brokerageLockIDs:
tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,
taskSpec.prodSourceLabel,
workQueue.queue_id, brokeragelockID,
True)
tmpStat, subChunks = splitter.doSplit(taskSpec, inputChunk, siteMapper)
tmpStat, pandaJobs, datasetToRegister, oldPandaIDs, parallelOutMap, outDsMap = gen.doGenerate(
taskSpec,
cloudName,
subChunks,
inputChunk,
tmpLog,
True,
splitter=splitter)
