# logger from pandacommon.pandalogger.PandaLogger import PandaLogger logger = PandaLogger().getLogger('JobGenerator') from pandajedi.jedicore.MsgWrapper import MsgWrapper tmpLog = MsgWrapper(logger) from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() from pandajedi.jediddm.DDMInterface import DDMInterface ddmIF = DDMInterface() ddmIF.setupInterface() from pandajedi.jediorder.JobBroker import JobBroker from pandajedi.jediorder.JobSplitter import JobSplitter from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool from pandajedi.jediorder.TaskSetupper import TaskSetupper import sys jediTaskID = int(sys.argv[1]) datasetIDs = None if len(sys.argv) > 2: datasetIDs = [int(sys.argv[2])]
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() vo = 'atlas' prodSourceLabel = 'managed' # get SiteMapper siteMapper = tbIF.getSiteMapper() wqMap = tbIF.getWorkQueueMap() tmpSt,jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo,prodSourceLabel) from pandajedi.jedithrottle.JobThrottler import JobThrottler jt = JobThrottler(vo,prodSourceLabel) jt.initialize(tbIF) print jt.toBeThrottled(vo,'US',wqMap.getQueueWithID(3),jobStat)
import sys try: metaID = sys.argv[1] except: metaID = None import json import uuid from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() reqIdx = 100000000 dsName = 'mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00' logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4()) taskParamMap = {} taskParamMap['nFilesPerJob'] = 1 taskParamMap['nFiles'] = 2 taskParamMap['nEventsPerInputFile'] = 10000 taskParamMap['nEventsPerJob'] = 10000 #taskParamMap['nEvents'] = 25000 #taskParamMap['skipScout'] = True taskParamMap['taskName'] = str(uuid.uuid4()) taskParamMap['userName'] = '******' taskParamMap['vo'] = 'atlas' taskParamMap['taskPriority'] = 900 #taskParamMap['reqID'] = reqIdx taskParamMap['architecture'] = 'i686-slc5-gcc43-opt'
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() vo = 'atlas' prodSourceLabel = 'managed' # get SiteMapper siteMapper = tbIF.getSiteMapper() wqMap = tbIF.getWorkQueueMap() tmpSt, jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo, prodSourceLabel) from pandajedi.jediorder.JobThrottler import JobThrottler jt = JobThrottler(vo, prodSourceLabel) jt.initializeMods(tbIF) workQueue = wqMap.getQueueWithID(1) print jt.toBeThrottled(vo, workQueue.queue_type, 'ND', workQueue, jobStat)
def start(self): # start zombi cleaner ZombiCleaner().start() # setup DDM I/F ddmIF = DDMInterface() ddmIF.setupInterface() # setup TaskBuffer I/F taskBufferIF = JediTaskBufferInterface() taskBufferIF.setupInterface() # the list of JEDI knights knightList = [] # setup TaskRefiner for itemStr in jedi_config.taskrefine.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskRefiner', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup TaskBrokerage for itemStr in jedi_config.taskbroker.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskBroker', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup ContentsFeeder for itemStr in jedi_config.confeeder.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.ContentsFeeder', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup JobGenerator for itemStr in jedi_config.jobgen.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] cloud = items[3] if not isinstance(cloud,list): cloud = [cloud] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = ProcessWrapper(target=self.launcher, args=('pandajedi.jediorder.JobGenerator', child_conn,taskBufferIF,ddmIF, vo,plabel,cloud,True,True)) proc.start() knightList.append(proc) # setup PostProcessor for itemStr in jedi_config.postprocessor.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.PostProcessor', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup TaskCommando for itemStr in jedi_config.tcommando.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.TaskCommando', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # setup WatchDog for itemStr in jedi_config.watchdog.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process(target=self.launcher, args=('pandajedi.jediorder.WatchDog', child_conn,taskBufferIF,ddmIF, vo,plabel)) proc.start() knightList.append(proc) # check initial failures time.sleep(5) for knight in knightList: if not knight.is_alive(): timeNow = datetime.datetime.utcnow() print "{0} {1}: ERROR pid={2} died in initialization".format(str(timeNow), self.__class__.__name__, knight.pid) os.killpg(os.getpgrp(),signal.SIGKILL) # join for knight in knightList: knight.join()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() from pandajedi.jediddm.DDMInterface import DDMInterface ddmIF = DDMInterface() ddmIF.setupInterface() import multiprocessing from pandajedi.jediorder import PostProcessor parent_conn, child_conn = multiprocessing.Pipe() postProcessor = multiprocessing.Process(target=PostProcessor.launcher, args=(child_conn,tbIF,ddmIF)) postProcessor.start()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediorder.JobThrottler import JobThrottler tbIF = JediTaskBufferInterface() tbIF.setupInterface() vo = 'atlas' prodSourceLabel = 'managed' cloud = 'WORLD' # get SiteMapper siteMapper = tbIF.getSiteMapper() wqMap = tbIF.getWorkQueueMap() jt = JobThrottler(vo, prodSourceLabel) jt.initializeMods(tbIF) workQueues = wqMap.getAlignedQueueList(vo, prodSourceLabel) resource_types = tbIF.load_resource_types() for workQueue in workQueues: for resource_type in resource_types: print( jt.toBeThrottled(vo, prodSourceLabel, cloud, workQueue, resource_type.resource_name))
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface # Set up the task buffer interface tb_if = JediTaskBufferInterface() tb_if.setupInterface() # Get the site mapper site_mapper = tb_if.getSiteMapper() # Define a random site of panda sites panda_sites = ['ANALY_DESY-HH', 'ANALY_DESY-HH_TEST', 'WT2_Install', 'BNL_Test_2_CE_1', 'ANALY_IN2P3-CC-T2_RD', 'UKI-SOUTHGRID-OX-HEP_SL6', 'RU-Protvino-IHEP', 'ANALY_NICS_Kraken', 'ANALY_TRIUMF_HIMEM', 'TESTGLEXEC', 'ANALY_GRIF-LPNHE', 'UKI-SCOTGRID-GLASGOW_MCORE', 'BNL_PROD', 'RAL-LCG2_SL6'] # Test the getSiteInputStorageEndpointMap function # print tb_if.getSiteInputStorageEndpointMap(panda_sites, site_mapper) # for panda_site in panda_sites: # print panda_site # tmp_site_spec = site_mapper.getSite('ANALY_DESY-HH') # print '------------------- ddm -------------------' # print 'ddm_input: {0}, ddm_output: {1}'.format(tmp_site_spec.ddm_input, tmp_site_spec.ddm_output) # print '------------------- setokens values -------------------' # print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input.values(), # tmp_site_spec.setokens_output.values()) # print '------------------- setokens -------------------' # print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input, # tmp_site_spec.setokens_output) from pandaserver.dataservice import DataServiceUtils # noqa: E402 sites = site_mapper.getCloud('WORLD')['sites']
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() wqMap = tbIF.getWorkQueueMap()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() from pandajedi.jediddm.DDMInterface import DDMInterface ddmIF = DDMInterface() ddmIF.setupInterface() atlasIF = ddmIF.getInterface('atlas') from pandajedi.jedibrokerage.AtlasProdJobBroker import AtlasProdJobBroker br = AtlasProdJobBroker(atlasIF,tbIF,siteMapper) st,taskSpec = tbIF.getTaskWithID_JEDI(1) print br.doBrokerage(taskSpec,'US', #{'mc12_8TeV.167751.Sherpa_CT10_ZeeMassiveCBPt0_CVetoBVeto.recon.AOD.e1585_a159_a171_tid01225039_00':['AOD.01225039._002187.pool.root.1','AOD.01225039._002627.pool.root.1']}) {'data11_7TeV.00190256.physics_JetTauEtmiss.merge.NTUP_SMWZ.f408_m1007_p1035_tid01048572_00':['NTUP_SMWZ.01048572._000522.root.1']})
# logger from pandacommon.pandalogger.PandaLogger import PandaLogger from pandajedi.jedicore.MsgWrapper import MsgWrapper from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jedirefine import RefinerUtils from pandajedi.jediorder.TaskRefiner import TaskRefiner from pandajedi.jediddm.DDMInterface import DDMInterface import sys logger = PandaLogger().getLogger('TaskRefiner') tmpLog = MsgWrapper(logger) tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID) refiner = TaskRefiner(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel) refiner.initializeMods(tbIF, ddmIF) taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID) taskParamMap = RefinerUtils.decodeJSON(taskParam)
import sys try: metaID = sys.argv[1] except: metaID = None import json import uuid from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() reqIdx = 100000000 dsName = ( "mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00" ) logDatasetName = "panda.jeditest.log.{0}".format(uuid.uuid4()) taskParamMap = {} taskParamMap["nFilesPerJob"] = 1 taskParamMap["nFiles"] = 2 taskParamMap["nEventsPerInputFile"] = 10000 taskParamMap["nEventsPerJob"] = 10000 # taskParamMap['nEvents'] = 25000 # taskParamMap['skipScout'] = True taskParamMap["taskName"] = str(uuid.uuid4()) taskParamMap["userName"] = "******" taskParamMap["vo"] = "atlas"
def start(self): # start zombi cleaner ZombiCleaner().start() # setup DDM I/F ddmIF = DDMInterface() ddmIF.setupInterface() # setup TaskBuffer I/F taskBufferIF = JediTaskBufferInterface() taskBufferIF.setupInterface() # setup intra-node message queue broker proxies if hasattr(jedi_config, 'mq') and hasattr( jedi_config.mq, 'configFile') and jedi_config.mq.configFile: from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile) mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999) # the list of JEDI knights knightList = [] # setup TaskRefiner for itemStr in jedi_config.taskrefine.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskRefiner', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup TaskBrokerage for itemStr in jedi_config.taskbroker.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskBroker', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup ContentsFeeder for itemStr in jedi_config.confeeder.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.ContentsFeeder', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup JobGenerator for itemStr in jedi_config.jobgen.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] cloud = items[3] try: loop_cycle = items[4] except IndexError: loop_cycle = None if not isinstance(cloud, list): cloud = [cloud] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = ProcessWrapper(target=self.launcher, args=('pandajedi.jediorder.JobGenerator', child_conn, taskBufferIF, ddmIF, vo, plabel, cloud, True, True, loop_cycle)) proc.start() knightList.append(proc) # setup PostProcessor for itemStr in jedi_config.postprocessor.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.PostProcessor', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup TaskCommando for itemStr in jedi_config.tcommando.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.TaskCommando', child_conn, taskBufferIF, ddmIF, vo, plabel)) proc.start() knightList.append(proc) # setup WatchDog for itemStr in jedi_config.watchdog.procConfig.split(';'): items = self.convParams(itemStr) vo = items[0] plabel = items[1] nProc = items[2] subStr = items[3] if len(items) > 3 else None period = items[4] if len(items) > 4 else None for iproc in range(nProc): parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.WatchDog', child_conn, taskBufferIF, ddmIF, vo, plabel, subStr, period)) proc.start() knightList.append(proc) # setup JediMsgProcessor agent (only one system process) if hasattr(jedi_config, 'msgprocessor') and hasattr( jedi_config.msgprocessor, 'configFile') and jedi_config.msgprocessor.configFile: stop_event = multiprocessing.Event() self.stopEventList.append(stop_event) parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.JediMsgProcessor', stop_event)) proc.start() knightList.append(proc) # setup JediDaemon agent (only one system process) if hasattr(jedi_config, 'daemon') and hasattr( jedi_config.daemon, 'enable') and jedi_config.daemon.enable: parent_conn, child_conn = multiprocessing.Pipe() proc = multiprocessing.Process( target=self.launcher, args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF)) proc.start() knightList.append(proc) # check initial failures time.sleep(5) for knight in knightList: if not knight.is_alive(): timeNow = datetime.datetime.utcnow() print( "{0} {1}: ERROR pid={2} died in initialization".format( str(timeNow), self.__class__.__name__, knight.pid)) os.killpg(os.getpgrp(), signal.SIGKILL) # join for knight in knightList: knight.join()
# logger from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediddm.DDMInterface import DDMInterface from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock from pandajedi.jediorder.TaskSetupper import TaskSetupper import os import sys import socket tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) # get task attributes s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID) pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(), os.getpgrp()) vo = taskSpec.vo prodSourceLabel = taskSpec.prodSourceLabel workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare( taskSpec.workQueue_ID, taskSpec.gshare)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediddm.DDMInterface import DDMInterface from pandajedi.jediorder.JobBroker import JobBroker from pandajedi.jediorder.JobSplitter import JobSplitter from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool from pandajedi.jediorder.TaskSetupper import TaskSetupper import sys logger = PandaLogger().getLogger('JobGenerator') tmpLog = MsgWrapper(logger) tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) datasetIDs = None if len(sys.argv) > 2: datasetIDs = [int(sys.argv[2])] s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
# logger from pandacommon.pandalogger.PandaLogger import PandaLogger from pandajedi.jedicore.MsgWrapper import MsgWrapper from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jedirefine import RefinerUtils from pandajedi.jediorder.TaskBroker import TaskBroker from pandajedi.jediddm.DDMInterface import DDMInterface import sys logger = PandaLogger().getLogger('TaskBroker') tmpLog = MsgWrapper(logger) tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID, False) body = TaskBroker(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel) body.initializeMods(tbIF, ddmIF) taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID) taskParamMap = RefinerUtils.decodeJSON(taskParam)
import sys import uuid from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() from pandajedi.jedicore.JediTaskSpec import JediTaskSpec task = JediTaskSpec() task.jediTaskID = sys.argv[1] task.taskName = 'pandatest.{0}'.format(uuid.uuid4()) task.status = 'defined' task.userName = '******' task.vo = 'atlas' task.prodSourceLabel = 'managed' task.taskPriority = 100 task.currentPriority = task.taskPriority task.architecture = 'i686-slc5-gcc43-opt' task.transUses = 'Atlas-17.2.7' task.transHome = 'AtlasProduction-17.2.8.10' task.transPath = 'Reco_trf.py' task.workQueue_ID = 3 tbIF.insertTask_JEDI(task) from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec ds = JediDatasetSpec() ds.jediTaskID = task.jediTaskID
import sys try: metaID = sys.argv[1] except: metaID = None import json import uuid from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() inFileList = ['file1','file2','file3'] logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4()) taskParamMap = {} taskParamMap['nFilesPerJob'] = 1 taskParamMap['nFiles'] = len(inFileList) #taskParamMap['nEventsPerInputFile'] = 10000 #taskParamMap['nEventsPerJob'] = 10000 #taskParamMap['nEvents'] = 25000 taskParamMap['noInput'] = True taskParamMap['pfnList'] = inFileList #taskParamMap['mergeOutput'] = True taskParamMap['taskName'] = str(uuid.uuid4()) taskParamMap['userName'] = '******' taskParamMap['vo'] = 'nonatlas' taskParamMap['taskPriority'] = 900
# logger from pandacommon.pandalogger.PandaLogger import PandaLogger logger = PandaLogger().getLogger('JobGenerator') from pandajedi.jedicore.MsgWrapper import MsgWrapper tmpLog = MsgWrapper(logger) from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() from pandajedi.jediddm.DDMInterface import DDMInterface ddmIF = DDMInterface() ddmIF.setupInterface() from pandajedi.jediorder.JobBroker import JobBroker from pandajedi.jediorder.JobSplitter import JobSplitter from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool,ListWithLock from pandajedi.jediorder.TaskSetupper import TaskSetupper import sys jediTaskID = int(sys.argv[1]) try: datasetID = [int(sys.argv[2])] except: datasetID = None
# logger from pandacommon.pandalogger.PandaLogger import PandaLogger logger = PandaLogger().getLogger('JobGenerator') from pandajedi.jedicore.MsgWrapper import MsgWrapper tmpLog = MsgWrapper(logger) from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() from pandajedi.jediddm.DDMInterface import DDMInterface ddmIF = DDMInterface() ddmIF.setupInterface() from pandajedi.jediorder.JobBroker import JobBroker from pandajedi.jediorder.JobSplitter import JobSplitter from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock from pandajedi.jediorder.TaskSetupper import TaskSetupper import sys jediTaskID = int(sys.argv[1]) try: datasetID = [int(sys.argv[2])] except: datasetID = None
import sys try: testTaskType = sys.argv[1] except Exception: testTaskType = 'test' from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediddm.DDMInterface import DDMInterface import multiprocessing from pandajedi.jediorder import WatchDog tbIF = JediTaskBufferInterface() tbIF.setupInterface() ddmIF = DDMInterface() ddmIF.setupInterface() parent_conn, child_conn = multiprocessing.Pipe() watchDog = multiprocessing.Process(target=WatchDog.launcher, args=(child_conn,tbIF,ddmIF, 'atlas',testTaskType)) watchDog.start()