def process(self, msg_obj, decoded_data=None):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
tmp_log.info('start')
# parse
if decoded_data is None:
# json decode
try:
msg_dict = json.loads(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
msg_obj.data)
tmp_log.error(err_str)
raise
else:
msg_dict = decoded_data
# run
try:
tmp_log.debug('got message {0}'.format(msg_dict))
if msg_dict['msg_type'] == 'generate_job':
# get task to generate jobs
jediTaskID = int(msg_dict['taskid'])
s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
if not taskSpec:
tmp_log.debug('unknown task {}'.format(jediTaskID))
else:
# get WQ
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
# get inputs
tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
target_tasks=[jediTaskID])
if tmpList:
inputList = ListWithLock(tmpList)
# create thread
threadPool = ThreadPool()
siteMapper = self.tbIF.getSiteMapper()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(self.tbIF, self.ddmIF)
gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
True, taskSetupper, self.pid, workQueue, 'pjmsg',
None, None, None, False)
gen.start()
gen.join()
else:
tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
except Exception as e:
err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
tmp_log.error(err_str)
raise
# done
tmp_log.info('done')
prodSourceLabel,
cloudName,nFiles=10,simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
jobBroker.setTestMode(taskSpec.vo,taskSpec.prodSourceLabel)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None,
None,None,None,brokerageLockIDs)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
jobBroker.setLockID(taskSpec.vo,taskSpec.prodSourceLabel,123,0)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,taskSpec.prodSourceLabel)
if brokerageLockID != None:
brokerageLockIDs.append(brokerageLockID)
for brokeragelockID in brokerageLockIDs:
tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,taskSpec.prodSourceLabel,workQueue.queue_id,
brokeragelockID,True)
tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
tmpStat,pandaJobs,datasetToRegister,oldPandaIDs,parallelOutMap,outDsMap = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True,
splitter=splitter)
workQueue,
prodSourceLabel,
cloudName,
nFiles=10,
simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetIDs)
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
for dummyID, tmpList in tmpListList:
for taskSpec, cloudName, inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo, taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo), tbIF)
splitter = JobSplitter()
gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
False, taskSetupper, None, None, 'dummy',
None, None)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
tmpLog)
tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
inputChunk, taskParamMap)
#tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
prodSourceLabel,
cloudName,nFiles=10,simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,None)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
tmpStat,pandaJobs,datasetToRegister,oldPandaIDs = gen.doGenerate(taskSpec,cloudName,subChunks,inputChunk,tmpLog,True)
if taskSpec.useEventService():
pandaJobs = gen.increaseEventServiceConsumers(pandaJobs,taskSpec.getNumEventServiceConsumer())
# get typical number of files
#typicalNumFilesMap = tbIF.getTypicalNumInput_JEDI(vo,prodSourceLabel,workQueue,
# useResultCache=600)
typicalNumFilesMap = {}
tmpListList = tbIF.getTasksToBeProcessed_JEDI(None,vo,workQueue,
prodSourceLabel,
cloudName,nFiles=10,simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetIDs)
taskSetupper = TaskSetupper(vo,prodSourceLabel)
taskSetupper.initializeMods(tbIF,ddmIF)
for dummyID,tmpList in tmpListList:
for taskSpec,cloudName,inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo,taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo),tbIF)
splitter = JobSplitter()
gen = JobGeneratorThread(None,threadPool,tbIF,ddmIF,siteMapper,False,taskSetupper,'pid')
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat,taskParamMap = gen.readTaskParams(taskSpec,taskParamMap,tmpLog)
tmpStat,inputChunk = jobBroker.doBrokerage(taskSpec,cloudName,inputChunk,taskParamMap)
#tmpStat,subChunks = splitter.doSplit(taskSpec,inputChunk,siteMapper)
simTasks=[jediTaskID],
fullSimulation=True,
typicalNumFilesMap=typicalNumFilesMap,
simDatasets=datasetID)
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
for dummyID, tmpList in tmpListList:
for taskSpec, cloudName, inputChunk in tmpList:
jobBroker = JobBroker(taskSpec.vo, taskSpec.prodSourceLabel)
tmpStat = jobBroker.initializeMods(ddmIF.getInterface(vo), tbIF)
jobBroker.setTestMode(taskSpec.vo, taskSpec.prodSourceLabel)
splitter = JobSplitter()
gen = JobGeneratorThread(None, threadPool, tbIF, ddmIF, siteMapper,
False, taskSetupper, None, None, None, None,
brokerageLockIDs, False)
taskParamMap = None
if taskSpec.useLimitedSites():
tmpStat, taskParamMap = gen.readTaskParams(taskSpec, taskParamMap,
tmpLog)
jobBroker.setLockID(taskSpec.vo, taskSpec.prodSourceLabel, 123, 0)
tmpStat, inputChunk = jobBroker.doBrokerage(taskSpec, cloudName,
inputChunk, taskParamMap)
brokerageLockID = jobBroker.getBaseLockID(taskSpec.vo,
taskSpec.prodSourceLabel)
if brokerageLockID != None:
brokerageLockIDs.append(brokerageLockID)
for brokeragelockID in brokerageLockIDs:
tbIF.unlockProcessWithPID_JEDI(taskSpec.vo,
jediTaskID = int(sys.argv[1])
# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
taskSpec.workQueue_ID, taskSpec.gshare)
# get inputs
tmpList = tbIF.getTasksToBeProcessed_JEDI(pid,
None,
workQueue,
None,
None,
nFiles=1000,
target_tasks=[jediTaskID])
inputList = ListWithLock(tmpList)
# create thread
threadPool = ThreadPool()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
gen = JobGeneratorThread(inputList, threadPool, tbIF, ddmIF, siteMapper, True,
taskSetupper, pid, workQueue, 'sgen', None, None,
None, False)
gen.start()
gen.join()