コード例 #1
0
 def process(self, msg_obj, decoded_data=None):
     # logger
     tmp_log = logger_utils.make_logger(base_logger, method_name='process')
     # start
     tmp_log.info('start')
     # parse
     if decoded_data is None:
         # json decode
         try:
             msg_dict = json.loads(msg_obj.data)
         except Exception as e:
             err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
                                                                                      msg_obj.data)
             tmp_log.error(err_str)
             raise
     else:
         msg_dict = decoded_data
     # run
     try:
         tmp_log.debug('got message {0}'.format(msg_dict))
         if msg_dict['msg_type'] == 'generate_job':
             # get task to generate jobs
             jediTaskID = int(msg_dict['taskid'])
             s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
             if not taskSpec:
                 tmp_log.debug('unknown task {}'.format(jediTaskID))
             else:
                 # get WQ
                 vo = taskSpec.vo
                 prodSourceLabel = taskSpec.prodSourceLabel
                 workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
                 # get inputs
                 tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
                                                                target_tasks=[jediTaskID])
                 if tmpList:
                     inputList = ListWithLock(tmpList)
                     # create thread
                     threadPool = ThreadPool()
                     siteMapper = self.tbIF.getSiteMapper()
                     taskSetupper = TaskSetupper(vo, prodSourceLabel)
                     taskSetupper.initializeMods(self.tbIF, self.ddmIF)
                     gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
                                              True, taskSetupper, self.pid, workQueue, 'pjmsg',
                                              None, None, None, False)
                     gen.start()
                     gen.join()
         else:
             tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
     except Exception as e:
         err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
         tmp_log.error(err_str)
         raise
     # done
     tmp_log.info('done')
コード例 #2
0
                                              prodSourceLabel,
                                              cloudName,nFiles=10,simTasks=[jediTaskID],
                                              fullSimulation=True,
                                              typicalNumFilesMap=typicalNumFilesMap,
                                              simDatasets=datasetID)

taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)

for dummyID,tmpList in tmpListList:
    for taskSpec,cloudName,inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
        jobBroker.setTestMode(taskSpec.vo,taskSpec.prodSourceLabel)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None,
                                 None,None,None,brokerageLockIDs)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
        jobBroker.setLockID(taskSpec.vo,taskSpec.prodSourceLabel,123,0)
        tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
        brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,taskSpec.prodSourceLabel)
        if brokerageLockID != None:
            brokerageLockIDs.append(brokerageLockID)
        for brokeragelockID in brokerageLockIDs:
            tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,taskSpec.prodSourceLabel,workQueue.queue_id,
                                           brokeragelockID,True)
        tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
        tmpStat,pandaJobs,datasetToRegister,oldPandaIDs,parallelOutMap,outDsMap = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True,
                                                                                                 splitter=splitter)
コード例 #3
0
    workQueue,
    prodSourceLabel,
    cloudName,
    nFiles=10,
    simTasks=[jediTaskID],
    fullSimulation=True,
    typicalNumFilesMap=typicalNumFilesMap,
    simDatasets=datasetIDs)

taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)

for dummyID, tmpList in tmpListList:
    for taskSpec, cloudName, inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo, taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo), tbIF)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
                                 False, taskSetupper, None, None, 'dummy',
                                 None, None)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
                                                       tmpLog)

        tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
                                                    inputChunk, taskParamMap)

        #tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
コード例 #4
0
tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
                                              prodSourceLabel,
                                              cloudName,nFiles=10,simTasks=[jediTaskID],
                                              fullSimulation=True,
                                              typicalNumFilesMap=typicalNumFilesMap,
                                              simDatasets=datasetID)

taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)

for dummyID,tmpList in tmpListList:
    for taskSpec,cloudName,inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)

        tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)

        tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)

        tmpStat,pandaJobs,datasetToRegister,oldPandaIDs = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True)
        if taskSpec.useEventService():
            pandaJobs = gen.increaseEventServiceConsumers(pandaJobs,taskSpec.getNumEventServiceConsumer())

コード例 #5
0
ファイル: brokerTest.py プロジェクト: lukewayne123/panda-jedi
# get typical number of files
#typicalNumFilesMap = tbIF.getTypicalNumInput_JEDI(vo,prodSourceLabel,workQueue,
#                                                  useResultCache=600)

typicalNumFilesMap = {}

tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
                                              prodSourceLabel,
                                              cloudName,nFiles=10,simTasks=[jediTaskID],
                                              fullSimulation=True,
                                              typicalNumFilesMap=typicalNumFilesMap,
                                              simDatasets=datasetIDs)

taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)

for dummyID,tmpList in tmpListList:
    for taskSpec,cloudName,inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,'pid')

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)

        tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)

        #tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
コード例 #6
0
    simTasks=[jediTaskID],
    fullSimulation=True,
    typicalNumFilesMap=typicalNumFilesMap,
    simDatasets=datasetID)

taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)

for dummyID, tmpList in tmpListList:
    for taskSpec, cloudName, inputChunk in tmpList:
        jobBroker = JobBroker(taskSpec.vo, taskSpec.prodSourceLabel)
        tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo), tbIF)
        jobBroker.setTestMode(taskSpec.vo, taskSpec.prodSourceLabel)
        splitter = JobSplitter()
        gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
                                 False, taskSetupper, None, None, None, None,
                                 brokerageLockIDs, False)

        taskParamMap = None
        if taskSpec.useLimitedSites():
            tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
                                                       tmpLog)
        jobBroker.setLockID(taskSpec.vo, taskSpec.prodSourceLabel, 123, 0)
        tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
                                                    inputChunk, taskParamMap)
        brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,
                                                  taskSpec.prodSourceLabel)
        if brokerageLockID != None:
            brokerageLockIDs.append(brokerageLockID)
        for brokeragelockID in brokerageLockIDs:
            tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,
コード例 #7
0
jediTaskID = int(sys.argv[1])

# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
                                os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
    taskSpec.workQueue_ID, taskSpec.gshare)

# get inputs
tmpList = tbIF.getTasksToBeProcessed_JEDI(pid,
                                          None,
                                          workQueue,
                                          None,
                                          None,
                                          nFiles=1000,
                                          target_tasks=[jediTaskID])
inputList = ListWithLock(tmpList)

# create thread
threadPool = ThreadPool()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
gen = JobGeneratorThread(inputList, threadPool, tbIF, ddmIF, siteMapper, True,
                         taskSetupper, pid, workQueue, 'sgen', None, None,
                         None, False)
gen.start()
gen.join()