import sys try: testTaskType = sys.argv[1] except Exception: testTaskType = 'test' from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediddm.DDMInterface import DDMInterface import multiprocessing from pandajedi.jediorder import WatchDog tbIF = JediTaskBufferInterface() tbIF.setupInterface() ddmIF = DDMInterface() ddmIF.setupInterface() parent_conn, child_conn = multiprocessing.Pipe() watchDog = multiprocessing.Process(target=WatchDog.launcher, args=(child_conn,tbIF,ddmIF, 'atlas',testTaskType)) watchDog.start()
def start(self): # start zombi cleaner ZombiCleaner().start() # setup DDM I/F ddmIF = DDMInterface() ddmIF.setupInterface() # setup TaskBuffer I/F taskBufferIF = JediTaskBufferInterface() taskBufferIF.setupInterface() # setup intra-node message queue broker proxies if hasattr(jedi_config, 'mq') and hasattr( jedi_config.mq, 'configFile') and jedi_config.mq.configFile: from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile) mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999) # the list of JEDI knights knightList = [] # setup TaskRefiner for itemStr in jedi_config.taskrefine.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskRefiner', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup TaskBrokerage for itemStr in jedi_config.taskbroker.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskBroker', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup ContentsFeeder for itemStr in jedi_config.confeeder.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.ContentsFeeder', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup JobGenerator for itemStr in jedi_config.jobgen.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] cloud = items[3] try: loop_cycle = items[4] except IndexError: loop_cycle = None if not isinstance(cloud, list): cloud = [cloud] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = ProcessWrapper(target=self.launcher, args=('pandajedi.jediorder.JobGenerator', child_conn, taskBufferIF, ddmIF, vo, plabel, cloud, True, True, loop_cycle)) proc.start() knightList.append(proc) # setup PostProcessor for itemStr in jedi_config.postprocessor.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.PostProcessor', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup TaskCommando for itemStr in jedi_config.tcommando.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskCommando', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup WatchDog for itemStr in jedi_config.watchdog.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] subStr = items[3] if len(items) > 3 else None period = items[4] if len(items) > 4 else None for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.WatchDog', child_conn, taskBufferIF, ddmIF, vo, plabel, subStr, period)) proc.start() knightList.append(proc) # setup JediMsgProcessor agent (only one system process) if hasattr(jedi_config, 'msgprocessor') and hasattr( jedi_config.msgprocessor, 'configFile') and jedi_config.msgprocessor.configFile: stop_event = multiprocessing.Event() self.stopEventList.append(stop_event) parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.JediMsgProcessor', stop_event)) proc.start() knightList.append(proc) # setup JediDaemon agent (only one system process) if hasattr(jedi_config, 'daemon') and hasattr( jedi_config.daemon, 'enable') and jedi_config.daemon.enable: parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF)) proc.start() knightList.append(proc) # check initial failures time.sleep(5) for knight in knightList: if not knight.is_alive(): timeNow = datetime.datetime.utcnow() print( "{0} {1}: ERROR pid={2} died in initialization".format( str(timeNow), self.__class__.__name__, knight.pid)) os.killpg(os.getpgrp(), signal.SIGKILL) # join for knight in knightList: knight.join()
def start(self): # start zombi cleaner ZombiCleaner().start() # setup DDM I/F ddmIF = DDMInterface() ddmIF.setupInterface() # setup TaskBuffer I/F taskBufferIF = JediTaskBufferInterface() taskBufferIF.setupInterface() # the list of JEDI knights knightList = [] # setup TaskRefiner for itemStr in jedi_config.taskrefine.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskRefiner', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup TaskBrokerage for itemStr in jedi_config.taskbroker.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskBroker', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup ContentsFeeder for itemStr in jedi_config.confeeder.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.ContentsFeeder', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup JobGenerator for itemStr in jedi_config.jobgen.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] cloud = items[3] if not isinstance(cloud,list): cloud = [cloud] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = ProcessWrapper(target=self.launcher, args=('pandajedi.jediorder.JobGenerator', child_conn,taskBufferIF,ddmIF, vo,plabel,cloud,True,True)) proc.start() knightList.append(proc) # setup PostProcessor for itemStr in jedi_config.postprocessor.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.PostProcessor', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup TaskCommando for itemStr in jedi_config.tcommando.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskCommando', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup WatchDog for itemStr in jedi_config.watchdog.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.WatchDog', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # check initial failures time.sleep(5) for knight in knightList: if not knight.is_alive(): timeNow = datetime.datetime.utcnow() print "{0} {1}: ERROR pid={2} died in initialization".format(str(timeNow), self.__class__.__name__, knight.pid) os.killpg(os.getpgrp(),signal.SIGKILL) # join for knight in knightList: knight.join()
def initialize(self): BaseMsgProcPlugin.initialize(self) self.ddmIF = DDMInterface() self.ddmIF.setupInterface() self.pid = '{0}-{1}_{2}-pjmsg'.format(socket.getfqdn().split('.')[0], os.getpid(), os.getpgrp())