Esempio n. 1
0
 def process(self, msg_obj, decoded_data=None):
     # logger
     tmp_log = logger_utils.make_logger(base_logger, method_name='process')
     # start
     tmp_log.info('start')
     # parse
     if decoded_data is None:
         # json decode
         try:
             msg_dict = json.loads(msg_obj.data)
         except Exception as e:
             err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
                                                                                      msg_obj.data)
             tmp_log.error(err_str)
             raise
     else:
         msg_dict = decoded_data
     # run
     try:
         tmp_log.debug('got message {0}'.format(msg_dict))
         if msg_dict['msg_type'] == 'generate_job':
             # get task to generate jobs
             jediTaskID = int(msg_dict['taskid'])
             s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
             if not taskSpec:
                 tmp_log.debug('unknown task {}'.format(jediTaskID))
             else:
                 # get WQ
                 vo = taskSpec.vo
                 prodSourceLabel = taskSpec.prodSourceLabel
                 workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
                 # get inputs
                 tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
                                                                target_tasks=[jediTaskID])
                 if tmpList:
                     inputList = ListWithLock(tmpList)
                     # create thread
                     threadPool = ThreadPool()
                     siteMapper = self.tbIF.getSiteMapper()
                     taskSetupper = TaskSetupper(vo, prodSourceLabel)
                     taskSetupper.initializeMods(self.tbIF, self.ddmIF)
                     gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
                                              True, taskSetupper, self.pid, workQueue, 'pjmsg',
                                              None, None, None, False)
                     gen.start()
                     gen.join()
         else:
             tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
     except Exception as e:
         err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
         tmp_log.error(err_str)
         raise
     # done
     tmp_log.info('done')
Esempio n. 2
0
jediTaskID = int(sys.argv[1])

# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
                                os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
    taskSpec.workQueue_ID, taskSpec.gshare)

# get inputs
tmpList = tbIF.getTasksToBeProcessed_JEDI(pid,
                                          None,
                                          workQueue,
                                          None,
                                          None,
                                          nFiles=1000,
                                          target_tasks=[jediTaskID])
inputList = ListWithLock(tmpList)

# create thread
threadPool = ThreadPool()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(tbIF, ddmIF)
gen = JobGeneratorThread(inputList, threadPool, tbIF, ddmIF, siteMapper, True,
                         taskSetupper, pid, workQueue, 'sgen', None, None,
                         None, False)
gen.start()
gen.join()