def process(self, msg_obj, decoded_data=None):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
tmp_log.info('start')
# parse
if decoded_data is None:
# json decode
try:
msg_dict = json.loads(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
msg_obj.data)
tmp_log.error(err_str)
raise
else:
msg_dict = decoded_data
# run
try:
tmp_log.debug('got message {0}'.format(msg_dict))
if msg_dict['msg_type'] == 'generate_job':
# get task to generate jobs
jediTaskID = int(msg_dict['taskid'])
s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
if not taskSpec:
tmp_log.debug('unknown task {}'.format(jediTaskID))
else:
# get WQ
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
# get inputs
tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
target_tasks=[jediTaskID])
if tmpList:
inputList = ListWithLock(tmpList)
# create thread
threadPool = ThreadPool()
siteMapper = self.tbIF.getSiteMapper()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(self.tbIF, self.ddmIF)
gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
True, taskSetupper, self.pid, workQueue, 'pjmsg',
None, None, None, False)
gen.start()
gen.join()
else:
tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
except Exception as e:
err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
tmp_log.error(err_str)
raise
# done
tmp_log.info('done')
jediTaskID = int(sys.argv[1])
# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
taskSpec.workQueue_ID, taskSpec.gshare)
# get inputs
tmpList = tbIF.getTasksToBeProcessed_JEDI(pid,
None,
workQueue,
None,
None,
nFiles=1000,
target_tasks=[jediTaskID])
inputList = ListWithLock(tmpList)
# create thread
threadPool = ThreadPool()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
gen = JobGeneratorThread(inputList, threadPool, tbIF, ddmIF, siteMapper, True,
taskSetupper, pid, workQueue, 'sgen', None, None,
None, False)
gen.start()
gen.join()