Ejemplo n.º 1
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
"""

from pandajedi.jediconfig import jedi_config

from pandajedi.jedicore import JediTaskBuffer

taskBufferIF = JediTaskBuffer.JediTaskBuffer(None)


from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()
ddmIF = ddmIF.getInterface("atlas")

datasetName = "valid1.159025.ParticleGenerator_gamma_E100.recon.AOD.e3099_s2082_r6012_tid04635343_00"
lostFiles = set(["AOD.04635343._000012.pool.root.1", "AOD.04635343._000015.pool.root.1"])


def resetStatusForLostFileRecovery(datasetName, lostFiles):
    # get jeditaskid
    varMap = {}
    varMap[":type1"] = "log"
    varMap[":type2"] = "output"
    varMap[":name1"] = datasetName
    varMap[":name2"] = datasetName.split(":")[-1]
    sqlGI = "SELECT jediTaskID,datasetID FROM {0}.JEDI_Datasets ".format(jedi_config.db.schemaJEDI)
Ejemplo n.º 2
0
import sys
try:
    testTaskType = sys.argv[1]
except Exception:
    testTaskType = 'test'

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface

import multiprocessing

from pandajedi.jediorder import WatchDog

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()


ddmIF = DDMInterface()
ddmIF.setupInterface()


parent_conn, child_conn = multiprocessing.Pipe()

watchDog = multiprocessing.Process(target=WatchDog.launcher,
                                   args=(child_conn,tbIF,ddmIF,
                                         'atlas',testTaskType))
watchDog.start()
Ejemplo n.º 3
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

import multiprocessing

from pandajedi.jediorder import PostProcessor

parent_conn, child_conn = multiprocessing.Pipe()

postProcessor = multiprocessing.Process(target=PostProcessor.launcher,
                                        args=(child_conn,tbIF,ddmIF))
postProcessor.start()
Ejemplo n.º 4
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool,ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys
jediTaskID = int(sys.argv[1])
try:
    datasetID = [int(sys.argv[2])]
except:
    datasetID = None
Ejemplo n.º 5
0
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys

logger = PandaLogger().getLogger('JobGenerator')
tmpLog = MsgWrapper(logger)

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

ddmIF = DDMInterface()
ddmIF.setupInterface()

jediTaskID = int(sys.argv[1])

datasetIDs = None
if len(sys.argv) > 2:
    datasetIDs = [int(sys.argv[2])]

s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)

cloudName = taskSpec.cloud
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
queueID = taskSpec.workQueue_ID
gshare_name = taskSpec.gshare
Ejemplo n.º 6
0
"""
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
"""

from pandajedi.jediconfig import jedi_config

from pandajedi.jedicore import JediTaskBuffer

taskBufferIF = JediTaskBuffer.JediTaskBuffer(None)

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()
ddmIF = ddmIF.getInterface('atlas')

datasetName = 'valid1.159025.ParticleGenerator_gamma_E100.recon.AOD.e3099_s2082_r6012_tid04635343_00'
lostFiles = set(
    ['AOD.04635343._000012.pool.root.1', 'AOD.04635343._000015.pool.root.1'])


def resetStatusForLostFileRecovery(datasetName, lostFiles):
    # get jeditaskid
    varMap = {}
    varMap[':type1'] = 'log'
    varMap[':type2'] = 'output'
    varMap[':name1'] = datasetName
    varMap[':name2'] = datasetName.split(':')[-1]
Ejemplo n.º 7
0
 def start(self):
     # start zombi cleaner
     ZombiCleaner().start()
     # setup DDM I/F
     ddmIF = DDMInterface()        
     ddmIF.setupInterface()
     # setup TaskBuffer I/F
     taskBufferIF = JediTaskBufferInterface()
     taskBufferIF.setupInterface()
     # the list of JEDI knights
     knightList = []
     # setup TaskRefiner
     for itemStr in jedi_config.taskrefine.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskRefiner',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup TaskBrokerage
     for itemStr in jedi_config.taskbroker.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskBroker',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup ContentsFeeder
     for itemStr in jedi_config.confeeder.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.ContentsFeeder',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup JobGenerator
     for itemStr in jedi_config.jobgen.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         cloud  = items[3]
         if not isinstance(cloud,list):
             cloud = [cloud]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = ProcessWrapper(target=self.launcher,
                                   args=('pandajedi.jediorder.JobGenerator',
                                         child_conn,taskBufferIF,ddmIF,
                                         vo,plabel,cloud,True,True))
             proc.start()
             knightList.append(proc)
     # setup PostProcessor
     for itemStr in jedi_config.postprocessor.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.PostProcessor',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup TaskCommando
     for itemStr in jedi_config.tcommando.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskCommando',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup WatchDog
     for itemStr in jedi_config.watchdog.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.WatchDog',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # check initial failures
     time.sleep(5)
     for knight in knightList:
         if not knight.is_alive():
             timeNow = datetime.datetime.utcnow()
             print "{0} {1}: ERROR    pid={2} died in initialization".format(str(timeNow),
                                                                             self.__class__.__name__,
                                                                             knight.pid)
             os.killpg(os.getpgrp(),signal.SIGKILL)
     # join
     for knight in knightList:    
         knight.join()
Ejemplo n.º 8
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()
atlasIF = ddmIF.getInterface('atlas')

from pandajedi.jedibrokerage.AtlasProdJobBroker import AtlasProdJobBroker
br = AtlasProdJobBroker(atlasIF,tbIF,siteMapper)

st,taskSpec = tbIF.getTaskWithID_JEDI(1)
print br.doBrokerage(taskSpec,'US',
  #{'mc12_8TeV.167751.Sherpa_CT10_ZeeMassiveCBPt0_CVetoBVeto.recon.AOD.e1585_a159_a171_tid01225039_00':['AOD.01225039._002187.pool.root.1','AOD.01225039._002627.pool.root.1']})
  {'data11_7TeV.00190256.physics_JetTauEtmiss.merge.NTUP_SMWZ.f408_m1007_p1035_tid01048572_00':['NTUP_SMWZ.01048572._000522.root.1']})
Ejemplo n.º 9
0
    def start(self):
        # start zombi cleaner
        ZombiCleaner().start()
        # setup DDM I/F
        ddmIF = DDMInterface()
        ddmIF.setupInterface()
        # setup TaskBuffer I/F
        taskBufferIF = JediTaskBufferInterface()
        taskBufferIF.setupInterface()
        # setup intra-node message queue broker proxies
        if hasattr(jedi_config, 'mq') and hasattr(
                jedi_config.mq, 'configFile') and jedi_config.mq.configFile:
            from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent
            mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile)
            mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999)
        # the list of JEDI knights
        knightList = []
        # setup TaskRefiner
        for itemStr in jedi_config.taskrefine.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskRefiner', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup TaskBrokerage
        for itemStr in jedi_config.taskbroker.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskBroker', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup ContentsFeeder
        for itemStr in jedi_config.confeeder.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.ContentsFeeder', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup JobGenerator
        for itemStr in jedi_config.jobgen.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            cloud = items[3]
            try:
                loop_cycle = items[4]
            except IndexError:
                loop_cycle = None

            if not isinstance(cloud, list):
                cloud = [cloud]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = ProcessWrapper(target=self.launcher,
                                      args=('pandajedi.jediorder.JobGenerator',
                                            child_conn, taskBufferIF, ddmIF,
                                            vo, plabel, cloud, True, True,
                                            loop_cycle))
                proc.start()
                knightList.append(proc)
        # setup PostProcessor
        for itemStr in jedi_config.postprocessor.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.PostProcessor', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup TaskCommando
        for itemStr in jedi_config.tcommando.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskCommando', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup WatchDog
        for itemStr in jedi_config.watchdog.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            subStr = items[3] if len(items) > 3 else None
            period = items[4] if len(items) > 4 else None
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.WatchDog', child_conn,
                          taskBufferIF, ddmIF, vo, plabel, subStr, period))
                proc.start()
                knightList.append(proc)
        # setup JediMsgProcessor agent (only one system process)
        if hasattr(jedi_config, 'msgprocessor') and hasattr(
                jedi_config.msgprocessor,
                'configFile') and jedi_config.msgprocessor.configFile:
            stop_event = multiprocessing.Event()
            self.stopEventList.append(stop_event)
            parent_conn, child_conn = multiprocessing.Pipe()
            proc = multiprocessing.Process(
                target=self.launcher,
                args=('pandajedi.jediorder.JediMsgProcessor', stop_event))
            proc.start()
            knightList.append(proc)
        # setup JediDaemon agent (only one system process)
        if hasattr(jedi_config, 'daemon') and hasattr(
                jedi_config.daemon, 'enable') and jedi_config.daemon.enable:
            parent_conn, child_conn = multiprocessing.Pipe()
            proc = multiprocessing.Process(
                target=self.launcher,
                args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF))
            proc.start()
            knightList.append(proc)
        # check initial failures
        time.sleep(5)
        for knight in knightList:
            if not knight.is_alive():
                timeNow = datetime.datetime.utcnow()
                print(
                    "{0} {1}: ERROR    pid={2} died in initialization".format(
                        str(timeNow), self.__class__.__name__, knight.pid))
                os.killpg(os.getpgrp(), signal.SIGKILL)
        # join
        for knight in knightList:
            knight.join()
Ejemplo n.º 10
0
 def initialize(self):
     BaseMsgProcPlugin.initialize(self)
     self.ddmIF = DDMInterface()
     self.ddmIF.setupInterface()
     self.pid = '{0}-{1}_{2}-pjmsg'.format(socket.getfqdn().split('.')[0], os.getpid(), os.getpgrp())
Ejemplo n.º 11
0
class PandaToJediMsgProcPlugin(BaseMsgProcPlugin):

    def initialize(self):
        BaseMsgProcPlugin.initialize(self)
        self.ddmIF = DDMInterface()
        self.ddmIF.setupInterface()
        self.pid = '{0}-{1}_{2}-pjmsg'.format(socket.getfqdn().split('.')[0], os.getpid(), os.getpgrp())

    def process(self, msg_obj, decoded_data=None):
        # logger
        tmp_log = logger_utils.make_logger(base_logger, method_name='process')
        # start
        tmp_log.info('start')
        # parse
        if decoded_data is None:
            # json decode
            try:
                msg_dict = json.loads(msg_obj.data)
            except Exception as e:
                err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
                                                                                         msg_obj.data)
                tmp_log.error(err_str)
                raise
        else:
            msg_dict = decoded_data
        # run
        try:
            tmp_log.debug('got message {0}'.format(msg_dict))
            if msg_dict['msg_type'] == 'generate_job':
                # get task to generate jobs
                jediTaskID = int(msg_dict['taskid'])
                s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
                if not taskSpec:
                    tmp_log.debug('unknown task {}'.format(jediTaskID))
                else:
                    # get WQ
                    vo = taskSpec.vo
                    prodSourceLabel = taskSpec.prodSourceLabel
                    workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
                    # get inputs
                    tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
                                                                   target_tasks=[jediTaskID])
                    if tmpList:
                        inputList = ListWithLock(tmpList)
                        # create thread
                        threadPool = ThreadPool()
                        siteMapper = self.tbIF.getSiteMapper()
                        taskSetupper = TaskSetupper(vo, prodSourceLabel)
                        taskSetupper.initializeMods(self.tbIF, self.ddmIF)
                        gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
                                                 True, taskSetupper, self.pid, workQueue, 'pjmsg',
                                                 None, None, None, False)
                        gen.start()
                        gen.join()
            else:
                tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
        except Exception as e:
            err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
            tmp_log.error(err_str)
            raise
        # done
        tmp_log.info('done')