import sys
try:
testTaskType = sys.argv[1]
except Exception:
testTaskType = 'test'
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface
import multiprocessing
from pandajedi.jediorder import WatchDog
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
ddmIF = DDMInterface()
ddmIF.setupInterface()
parent_conn, child_conn = multiprocessing.Pipe()
watchDog = multiprocessing.Process(target=WatchDog.launcher,
args=(child_conn,tbIF,ddmIF,
'atlas',testTaskType))
watchDog.start()
def start(self):
# start zombi cleaner
ZombiCleaner().start()
# setup DDM I/F
ddmIF = DDMInterface()
ddmIF.setupInterface()
# setup TaskBuffer I/F
taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
# setup intra-node message queue broker proxies
if hasattr(jedi_config, 'mq') and hasattr(
jedi_config.mq, 'configFile') and jedi_config.mq.configFile:
from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent
mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile)
mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999)
# the list of JEDI knights
knightList = []
# setup TaskRefiner
for itemStr in jedi_config.taskrefine.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskRefiner', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup TaskBrokerage
for itemStr in jedi_config.taskbroker.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskBroker', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup ContentsFeeder
for itemStr in jedi_config.confeeder.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.ContentsFeeder', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup JobGenerator
for itemStr in jedi_config.jobgen.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
cloud = items[3]
try:
loop_cycle = items[4]
except IndexError:
loop_cycle = None
if not isinstance(cloud, list):
cloud = [cloud]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = ProcessWrapper(target=self.launcher,
args=('pandajedi.jediorder.JobGenerator',
child_conn, taskBufferIF, ddmIF,
vo, plabel, cloud, True, True,
loop_cycle))
proc.start()
knightList.append(proc)
# setup PostProcessor
for itemStr in jedi_config.postprocessor.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.PostProcessor', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup TaskCommando
for itemStr in jedi_config.tcommando.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskCommando', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup WatchDog
for itemStr in jedi_config.watchdog.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
subStr = items[3] if len(items) > 3 else None
period = items[4] if len(items) > 4 else None
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.WatchDog', child_conn,
taskBufferIF, ddmIF, vo, plabel, subStr, period))
proc.start()
knightList.append(proc)
# setup JediMsgProcessor agent (only one system process)
if hasattr(jedi_config, 'msgprocessor') and hasattr(
jedi_config.msgprocessor,
'configFile') and jedi_config.msgprocessor.configFile:
stop_event = multiprocessing.Event()
self.stopEventList.append(stop_event)
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.JediMsgProcessor', stop_event))
proc.start()
knightList.append(proc)
# setup JediDaemon agent (only one system process)
if hasattr(jedi_config, 'daemon') and hasattr(
jedi_config.daemon, 'enable') and jedi_config.daemon.enable:
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF))
proc.start()
knightList.append(proc)
# check initial failures
time.sleep(5)
for knight in knightList:
if not knight.is_alive():
timeNow = datetime.datetime.utcnow()
print(
"{0} {1}: ERROR pid={2} died in initialization".format(
str(timeNow), self.__class__.__name__, knight.pid))
os.killpg(os.getpgrp(), signal.SIGKILL)
# join
for knight in knightList:
knight.join()
def start(self):
# start zombi cleaner
ZombiCleaner().start()
# setup DDM I/F
ddmIF = DDMInterface()
ddmIF.setupInterface()
# setup TaskBuffer I/F
taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
# the list of JEDI knights
knightList = []
# setup TaskRefiner
for itemStr in jedi_config.taskrefine.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskRefiner',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup TaskBrokerage
for itemStr in jedi_config.taskbroker.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskBroker',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup ContentsFeeder
for itemStr in jedi_config.confeeder.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.ContentsFeeder',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup JobGenerator
for itemStr in jedi_config.jobgen.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
cloud = items[3]
if not isinstance(cloud,list):
cloud = [cloud]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = ProcessWrapper(target=self.launcher,
args=('pandajedi.jediorder.JobGenerator',
child_conn,taskBufferIF,ddmIF,
vo,plabel,cloud,True,True))
proc.start()
knightList.append(proc)
# setup PostProcessor
for itemStr in jedi_config.postprocessor.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.PostProcessor',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup TaskCommando
for itemStr in jedi_config.tcommando.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskCommando',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup WatchDog
for itemStr in jedi_config.watchdog.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.WatchDog',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# check initial failures
time.sleep(5)
for knight in knightList:
if not knight.is_alive():
timeNow = datetime.datetime.utcnow()
print "{0} {1}: ERROR pid={2} died in initialization".format(str(timeNow),
self.__class__.__name__,
knight.pid)
os.killpg(os.getpgrp(),signal.SIGKILL)
# join
for knight in knightList:
knight.join()
def initialize(self):
BaseMsgProcPlugin.initialize(self)
self.ddmIF = DDMInterface()
self.ddmIF.setupInterface()
self.pid = '{0}-{1}_{2}-pjmsg'.format(socket.getfqdn().split('.')[0], os.getpid(), os.getpgrp())
