Ejemplo n.º 1
0
tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
                                              prodSourceLabel,
                                              cloudName,nFiles=10,simTasks=[jediTaskID],
                                              fullSimulation=True,
                                              typicalNumFilesMap=typicalNumFilesMap,
                                              simDatasets=datasetID)

taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)

for dummyID,tmpList in tmpListList:
    for taskSpec,cloudName,inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)

        tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)

        tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)

        tmpStat,pandaJobs,datasetToRegister,oldPandaIDs = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True)
        if taskSpec.useEventService():
            pandaJobs = gen.increaseEventServiceConsumers(pandaJobs,taskSpec.getNumEventServiceConsumer())

Ejemplo n.º 2
0
                                              typicalNumFilesMap=typicalNumFilesMap,
                                              simDatasets=datasetID)

taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)

for dummyID,tmpList in tmpListList:
    for taskSpec,cloudName,inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
        jobBroker.setTestMode(taskSpec.vo,taskSpec.prodSourceLabel)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None,
                                 None,None,None,brokerageLockIDs)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
        jobBroker.setLockID(taskSpec.vo,taskSpec.prodSourceLabel,123,0)
        tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
        brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,taskSpec.prodSourceLabel)
        if brokerageLockID != None:
            brokerageLockIDs.append(brokerageLockID)
        for brokeragelockID in brokerageLockIDs:
            tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,taskSpec.prodSourceLabel,workQueue.queue_id,
                                           brokeragelockID,True)
        tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
        tmpStat,pandaJobs,datasetToRegister,oldPandaIDs,parallelOutMap,outDsMap = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True,
                                                                                                 splitter=splitter)

Ejemplo n.º 3
0
        splitter = JobSplitter()
        gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
                                 False, taskSetupper, None, None, None, None,
                                 brokerageLockIDs, False)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
                                                       tmpLog)
        jobBroker.setLockID(taskSpec.vo, taskSpec.prodSourceLabel, 123, 0)
        tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
                                                    inputChunk, taskParamMap)
        brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,
                                                  taskSpec.prodSourceLabel)
        if brokerageLockID != None:
            brokerageLockIDs.append(brokerageLockID)
        for brokeragelockID in brokerageLockIDs:
            tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,
                                           taskSpec.prodSourceLabel,
                                           workQueue.queue_id, brokeragelockID,
                                           True)
        tmpStat, subChunks = splitter.doSplit(taskSpec, inputChunk, siteMapper)
        tmpStat, pandaJobs, datasetToRegister, oldPandaIDs, parallelOutMap, outDsMap = gen.doGenerate(
            taskSpec,
            cloudName,
            subChunks,
            inputChunk,
            tmpLog,
            True,
            splitter=splitter)