# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool
from pandajedi.jediorder.TaskSetupper import TaskSetupper
import sys
jediTaskID = int(sys.argv[1])
datasetIDs = None
if len(sys.argv) > 2:
datasetIDs = [int(sys.argv[2])]
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() vo = 'atlas' prodSourceLabel = 'managed' # get SiteMapper siteMapper = tbIF.getSiteMapper() wqMap = tbIF.getWorkQueueMap() tmpSt,jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo,prodSourceLabel) from pandajedi.jedithrottle.JobThrottler import JobThrottler jt = JobThrottler(vo,prodSourceLabel) jt.initialize(tbIF) print jt.toBeThrottled(vo,'US',wqMap.getQueueWithID(3),jobStat)
import sys
try:
metaID = sys.argv[1]
except:
metaID = None
import json
import uuid
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
reqIdx = 100000000
dsName = 'mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00'
logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4())
taskParamMap = {}
taskParamMap['nFilesPerJob'] = 1
taskParamMap['nFiles'] = 2
taskParamMap['nEventsPerInputFile'] = 10000
taskParamMap['nEventsPerJob'] = 10000
#taskParamMap['nEvents'] = 25000
#taskParamMap['skipScout'] = True
taskParamMap['taskName'] = str(uuid.uuid4())
taskParamMap['userName'] = '******'
taskParamMap['vo'] = 'atlas'
taskParamMap['taskPriority'] = 900
#taskParamMap['reqID'] = reqIdx
taskParamMap['architecture'] = 'i686-slc5-gcc43-opt'
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() vo = 'atlas' prodSourceLabel = 'managed' # get SiteMapper siteMapper = tbIF.getSiteMapper() wqMap = tbIF.getWorkQueueMap() tmpSt, jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo, prodSourceLabel) from pandajedi.jediorder.JobThrottler import JobThrottler jt = JobThrottler(vo, prodSourceLabel) jt.initializeMods(tbIF) workQueue = wqMap.getQueueWithID(1) print jt.toBeThrottled(vo, workQueue.queue_type, 'ND', workQueue, jobStat)
def start(self):
# start zombi cleaner
ZombiCleaner().start()
# setup DDM I/F
ddmIF = DDMInterface()
ddmIF.setupInterface()
# setup TaskBuffer I/F
taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
# the list of JEDI knights
knightList = []
# setup TaskRefiner
for itemStr in jedi_config.taskrefine.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskRefiner',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup TaskBrokerage
for itemStr in jedi_config.taskbroker.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskBroker',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup ContentsFeeder
for itemStr in jedi_config.confeeder.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.ContentsFeeder',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup JobGenerator
for itemStr in jedi_config.jobgen.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
cloud = items[3]
if not isinstance(cloud,list):
cloud = [cloud]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = ProcessWrapper(target=self.launcher,
args=('pandajedi.jediorder.JobGenerator',
child_conn,taskBufferIF,ddmIF,
vo,plabel,cloud,True,True))
proc.start()
knightList.append(proc)
# setup PostProcessor
for itemStr in jedi_config.postprocessor.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.PostProcessor',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup TaskCommando
for itemStr in jedi_config.tcommando.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.TaskCommando',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# setup WatchDog
for itemStr in jedi_config.watchdog.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(target=self.launcher,
args=('pandajedi.jediorder.WatchDog',
child_conn,taskBufferIF,ddmIF,
vo,plabel))
proc.start()
knightList.append(proc)
# check initial failures
time.sleep(5)
for knight in knightList:
if not knight.is_alive():
timeNow = datetime.datetime.utcnow()
print "{0} {1}: ERROR pid={2} died in initialization".format(str(timeNow),
self.__class__.__name__,
knight.pid)
os.killpg(os.getpgrp(),signal.SIGKILL)
# join
for knight in knightList:
knight.join()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
import multiprocessing
from pandajedi.jediorder import PostProcessor
parent_conn, child_conn = multiprocessing.Pipe()
postProcessor = multiprocessing.Process(target=PostProcessor.launcher,
args=(child_conn,tbIF,ddmIF))
postProcessor.start()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediorder.JobThrottler import JobThrottler
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
vo = 'atlas'
prodSourceLabel = 'managed'
cloud = 'WORLD'
# get SiteMapper
siteMapper = tbIF.getSiteMapper()
wqMap = tbIF.getWorkQueueMap()
jt = JobThrottler(vo, prodSourceLabel)
jt.initializeMods(tbIF)
workQueues = wqMap.getAlignedQueueList(vo, prodSourceLabel)
resource_types = tbIF.load_resource_types()
for workQueue in workQueues:
for resource_type in resource_types:
print(
jt.toBeThrottled(vo, prodSourceLabel, cloud, workQueue,
resource_type.resource_name))
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
# Set up the task buffer interface
tb_if = JediTaskBufferInterface()
tb_if.setupInterface()
# Get the site mapper
site_mapper = tb_if.getSiteMapper()
# Define a random site of panda sites
panda_sites = ['ANALY_DESY-HH', 'ANALY_DESY-HH_TEST', 'WT2_Install', 'BNL_Test_2_CE_1', 'ANALY_IN2P3-CC-T2_RD',
'UKI-SOUTHGRID-OX-HEP_SL6', 'RU-Protvino-IHEP', 'ANALY_NICS_Kraken', 'ANALY_TRIUMF_HIMEM',
'TESTGLEXEC', 'ANALY_GRIF-LPNHE', 'UKI-SCOTGRID-GLASGOW_MCORE', 'BNL_PROD', 'RAL-LCG2_SL6']
# Test the getSiteInputStorageEndpointMap function
# print tb_if.getSiteInputStorageEndpointMap(panda_sites, site_mapper)
# for panda_site in panda_sites:
# print panda_site
# tmp_site_spec = site_mapper.getSite('ANALY_DESY-HH')
# print '------------------- ddm -------------------'
# print 'ddm_input: {0}, ddm_output: {1}'.format(tmp_site_spec.ddm_input, tmp_site_spec.ddm_output)
# print '------------------- setokens values -------------------'
# print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input.values(),
# tmp_site_spec.setokens_output.values())
# print '------------------- setokens -------------------'
# print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input,
# tmp_site_spec.setokens_output)
from pandaserver.dataservice import DataServiceUtils # noqa: E402
sites = site_mapper.getCloud('WORLD')['sites']
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface tbIF = JediTaskBufferInterface() tbIF.setupInterface() wqMap = tbIF.getWorkQueueMap()
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
atlasIF = ddmIF.getInterface('atlas')
from pandajedi.jedibrokerage.AtlasProdJobBroker import AtlasProdJobBroker
br = AtlasProdJobBroker(atlasIF,tbIF,siteMapper)
st,taskSpec = tbIF.getTaskWithID_JEDI(1)
print br.doBrokerage(taskSpec,'US',
#{'mc12_8TeV.167751.Sherpa_CT10_ZeeMassiveCBPt0_CVetoBVeto.recon.AOD.e1585_a159_a171_tid01225039_00':['AOD.01225039._002187.pool.root.1','AOD.01225039._002627.pool.root.1']})
{'data11_7TeV.00190256.physics_JetTauEtmiss.merge.NTUP_SMWZ.f408_m1007_p1035_tid01048572_00':['NTUP_SMWZ.01048572._000522.root.1']})
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jedicore.MsgWrapper import MsgWrapper
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jedirefine import RefinerUtils
from pandajedi.jediorder.TaskRefiner import TaskRefiner
from pandajedi.jediddm.DDMInterface import DDMInterface
import sys
logger = PandaLogger().getLogger('TaskRefiner')
tmpLog = MsgWrapper(logger)
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
ddmIF = DDMInterface()
ddmIF.setupInterface()
jediTaskID = int(sys.argv[1])
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
refiner = TaskRefiner(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel)
refiner.initializeMods(tbIF, ddmIF)
taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID)
taskParamMap = RefinerUtils.decodeJSON(taskParam)
import sys
try:
metaID = sys.argv[1]
except:
metaID = None
import json
import uuid
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
reqIdx = 100000000
dsName = (
"mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00"
)
logDatasetName = "panda.jeditest.log.{0}".format(uuid.uuid4())
taskParamMap = {}
taskParamMap["nFilesPerJob"] = 1
taskParamMap["nFiles"] = 2
taskParamMap["nEventsPerInputFile"] = 10000
taskParamMap["nEventsPerJob"] = 10000
# taskParamMap['nEvents'] = 25000
# taskParamMap['skipScout'] = True
taskParamMap["taskName"] = str(uuid.uuid4())
taskParamMap["userName"] = "******"
taskParamMap["vo"] = "atlas"
def start(self):
# start zombi cleaner
ZombiCleaner().start()
# setup DDM I/F
ddmIF = DDMInterface()
ddmIF.setupInterface()
# setup TaskBuffer I/F
taskBufferIF = JediTaskBufferInterface()
taskBufferIF.setupInterface()
# setup intra-node message queue broker proxies
if hasattr(jedi_config, 'mq') and hasattr(
jedi_config.mq, 'configFile') and jedi_config.mq.configFile:
from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent
mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile)
mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999)
# the list of JEDI knights
knightList = []
# setup TaskRefiner
for itemStr in jedi_config.taskrefine.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskRefiner', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup TaskBrokerage
for itemStr in jedi_config.taskbroker.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskBroker', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup ContentsFeeder
for itemStr in jedi_config.confeeder.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.ContentsFeeder', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup JobGenerator
for itemStr in jedi_config.jobgen.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
cloud = items[3]
try:
loop_cycle = items[4]
except IndexError:
loop_cycle = None
if not isinstance(cloud, list):
cloud = [cloud]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = ProcessWrapper(target=self.launcher,
args=('pandajedi.jediorder.JobGenerator',
child_conn, taskBufferIF, ddmIF,
vo, plabel, cloud, True, True,
loop_cycle))
proc.start()
knightList.append(proc)
# setup PostProcessor
for itemStr in jedi_config.postprocessor.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.PostProcessor', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup TaskCommando
for itemStr in jedi_config.tcommando.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.TaskCommando', child_conn,
taskBufferIF, ddmIF, vo, plabel))
proc.start()
knightList.append(proc)
# setup WatchDog
for itemStr in jedi_config.watchdog.procConfig.split(';'):
items = self.convParams(itemStr)
vo = items[0]
plabel = items[1]
nProc = items[2]
subStr = items[3] if len(items) > 3 else None
period = items[4] if len(items) > 4 else None
for iproc in range(nProc):
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.WatchDog', child_conn,
taskBufferIF, ddmIF, vo, plabel, subStr, period))
proc.start()
knightList.append(proc)
# setup JediMsgProcessor agent (only one system process)
if hasattr(jedi_config, 'msgprocessor') and hasattr(
jedi_config.msgprocessor,
'configFile') and jedi_config.msgprocessor.configFile:
stop_event = multiprocessing.Event()
self.stopEventList.append(stop_event)
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.JediMsgProcessor', stop_event))
proc.start()
knightList.append(proc)
# setup JediDaemon agent (only one system process)
if hasattr(jedi_config, 'daemon') and hasattr(
jedi_config.daemon, 'enable') and jedi_config.daemon.enable:
parent_conn, child_conn = multiprocessing.Pipe()
proc = multiprocessing.Process(
target=self.launcher,
args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF))
proc.start()
knightList.append(proc)
# check initial failures
time.sleep(5)
for knight in knightList:
if not knight.is_alive():
timeNow = datetime.datetime.utcnow()
print(
"{0} {1}: ERROR pid={2} died in initialization".format(
str(timeNow), self.__class__.__name__, knight.pid))
os.killpg(os.getpgrp(), signal.SIGKILL)
# join
for knight in knightList:
knight.join()
# logger
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper
import os
import sys
import socket
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
ddmIF = DDMInterface()
ddmIF.setupInterface()
jediTaskID = int(sys.argv[1])
# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
taskSpec.workQueue_ID, taskSpec.gshare)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface
from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool
from pandajedi.jediorder.TaskSetupper import TaskSetupper
import sys
logger = PandaLogger().getLogger('JobGenerator')
tmpLog = MsgWrapper(logger)
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
ddmIF = DDMInterface()
ddmIF.setupInterface()
jediTaskID = int(sys.argv[1])
datasetIDs = None
if len(sys.argv) > 2:
datasetIDs = [int(sys.argv[2])]
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jedicore.MsgWrapper import MsgWrapper
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jedirefine import RefinerUtils
from pandajedi.jediorder.TaskBroker import TaskBroker
from pandajedi.jediddm.DDMInterface import DDMInterface
import sys
logger = PandaLogger().getLogger('TaskBroker')
tmpLog = MsgWrapper(logger)
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
ddmIF = DDMInterface()
ddmIF.setupInterface()
jediTaskID = int(sys.argv[1])
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID, False)
body = TaskBroker(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel)
body.initializeMods(tbIF, ddmIF)
taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID)
taskParamMap = RefinerUtils.decodeJSON(taskParam)
import sys
import uuid
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
from pandajedi.jedicore.JediTaskSpec import JediTaskSpec
task = JediTaskSpec()
task.jediTaskID = sys.argv[1]
task.taskName = 'pandatest.{0}'.format(uuid.uuid4())
task.status = 'defined'
task.userName = '******'
task.vo = 'atlas'
task.prodSourceLabel = 'managed'
task.taskPriority = 100
task.currentPriority = task.taskPriority
task.architecture = 'i686-slc5-gcc43-opt'
task.transUses = 'Atlas-17.2.7'
task.transHome = 'AtlasProduction-17.2.8.10'
task.transPath = 'Reco_trf.py'
task.workQueue_ID = 3
tbIF.insertTask_JEDI(task)
from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec
ds = JediDatasetSpec()
ds.jediTaskID = task.jediTaskID
import sys
try:
metaID = sys.argv[1]
except:
metaID = None
import json
import uuid
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
inFileList = ['file1','file2','file3']
logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4())
taskParamMap = {}
taskParamMap['nFilesPerJob'] = 1
taskParamMap['nFiles'] = len(inFileList)
#taskParamMap['nEventsPerInputFile'] = 10000
#taskParamMap['nEventsPerJob'] = 10000
#taskParamMap['nEvents'] = 25000
taskParamMap['noInput'] = True
taskParamMap['pfnList'] = inFileList
#taskParamMap['mergeOutput'] = True
taskParamMap['taskName'] = str(uuid.uuid4())
taskParamMap['userName'] = '******'
taskParamMap['vo'] = 'nonatlas'
taskParamMap['taskPriority'] = 900
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool,ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper
import sys
jediTaskID = int(sys.argv[1])
try:
datasetID = [int(sys.argv[2])]
except:
datasetID = None
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
siteMapper = tbIF.getSiteMapper()
from pandajedi.jediddm.DDMInterface import DDMInterface
ddmIF = DDMInterface()
ddmIF.setupInterface()
from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper
import sys
jediTaskID = int(sys.argv[1])
try:
datasetID = [int(sys.argv[2])]
except:
datasetID = None
import sys
try:
testTaskType = sys.argv[1]
except Exception:
testTaskType = 'test'
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface
import multiprocessing
from pandajedi.jediorder import WatchDog
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
ddmIF = DDMInterface()
ddmIF.setupInterface()
parent_conn, child_conn = multiprocessing.Pipe()
watchDog = multiprocessing.Process(target=WatchDog.launcher,
args=(child_conn,tbIF,ddmIF,
'atlas',testTaskType))
watchDog.start()
import sys
import uuid
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
from pandajedi.jedicore.JediTaskSpec import JediTaskSpec
task = JediTaskSpec()
task.jediTaskID = sys.argv[1]
task.taskName = 'pandatest.{0}'.format(uuid.uuid4())
task.status = 'defined'
task.userName = '******'
task.vo = 'atlas'
task.prodSourceLabel = 'managed'
task.taskPriority = 100
task.currentPriority = task.taskPriority
task.architecture = 'i686-slc5-gcc43-opt'
task.transUses = 'Atlas-17.2.7'
task.transHome = 'AtlasProduction-17.2.8.10'
task.transPath = 'Reco_trf.py'
task.workQueue_ID = 3
tbIF.insertTask_JEDI(task)
from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec
ds = JediDatasetSpec()
ds.jediTaskID = task.jediTaskID
