Ejemplo n.º 1
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys
jediTaskID = int(sys.argv[1])

datasetIDs = None
if len(sys.argv) > 2:
    datasetIDs = [int(sys.argv[2])]
Ejemplo n.º 2
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

vo = 'atlas'
prodSourceLabel = 'managed'

# get SiteMapper
siteMapper = tbIF.getSiteMapper()

wqMap = tbIF.getWorkQueueMap()

tmpSt,jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo,prodSourceLabel)

from pandajedi.jedithrottle.JobThrottler import JobThrottler

jt = JobThrottler(vo,prodSourceLabel)
jt.initialize(tbIF)

print jt.toBeThrottled(vo,'US',wqMap.getQueueWithID(3),jobStat)
Ejemplo n.º 3
0
import sys
try:
    metaID = sys.argv[1]
except:
    metaID = None
import json
import uuid

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

reqIdx = 100000000
dsName = 'mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00'
logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4())

taskParamMap = {}

taskParamMap['nFilesPerJob'] = 1
taskParamMap['nFiles'] = 2
taskParamMap['nEventsPerInputFile'] = 10000
taskParamMap['nEventsPerJob'] = 10000
#taskParamMap['nEvents'] = 25000
#taskParamMap['skipScout'] = True
taskParamMap['taskName'] = str(uuid.uuid4())
taskParamMap['userName'] = '******'
taskParamMap['vo'] = 'atlas'
taskParamMap['taskPriority'] = 900
#taskParamMap['reqID'] = reqIdx
taskParamMap['architecture'] = 'i686-slc5-gcc43-opt'
Ejemplo n.º 4
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

vo = 'atlas'
prodSourceLabel = 'managed'

# get SiteMapper
siteMapper = tbIF.getSiteMapper()

wqMap = tbIF.getWorkQueueMap()

tmpSt, jobStat = tbIF.getJobStatWithWorkQueuePerCloud_JEDI(vo, prodSourceLabel)

from pandajedi.jediorder.JobThrottler import JobThrottler

jt = JobThrottler(vo, prodSourceLabel)
jt.initializeMods(tbIF)

workQueue = wqMap.getQueueWithID(1)
print jt.toBeThrottled(vo, workQueue.queue_type, 'ND', workQueue, jobStat)
Ejemplo n.º 5
0
 def start(self):
     # start zombi cleaner
     ZombiCleaner().start()
     # setup DDM I/F
     ddmIF = DDMInterface()        
     ddmIF.setupInterface()
     # setup TaskBuffer I/F
     taskBufferIF = JediTaskBufferInterface()
     taskBufferIF.setupInterface()
     # the list of JEDI knights
     knightList = []
     # setup TaskRefiner
     for itemStr in jedi_config.taskrefine.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskRefiner',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup TaskBrokerage
     for itemStr in jedi_config.taskbroker.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskBroker',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup ContentsFeeder
     for itemStr in jedi_config.confeeder.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.ContentsFeeder',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup JobGenerator
     for itemStr in jedi_config.jobgen.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         cloud  = items[3]
         if not isinstance(cloud,list):
             cloud = [cloud]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = ProcessWrapper(target=self.launcher,
                                   args=('pandajedi.jediorder.JobGenerator',
                                         child_conn,taskBufferIF,ddmIF,
                                         vo,plabel,cloud,True,True))
             proc.start()
             knightList.append(proc)
     # setup PostProcessor
     for itemStr in jedi_config.postprocessor.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.PostProcessor',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup TaskCommando
     for itemStr in jedi_config.tcommando.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.TaskCommando',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # setup WatchDog
     for itemStr in jedi_config.watchdog.procConfig.split(';'):
         items = self.convParams(itemStr)
         vo     = items[0]
         plabel = items[1]
         nProc  = items[2]
         for iproc in range(nProc):
             parent_conn, child_conn = multiprocessing.Pipe()
             proc = multiprocessing.Process(target=self.launcher,
                                            args=('pandajedi.jediorder.WatchDog',
                                                  child_conn,taskBufferIF,ddmIF,
                                                  vo,plabel))
             proc.start()
             knightList.append(proc)
     # check initial failures
     time.sleep(5)
     for knight in knightList:
         if not knight.is_alive():
             timeNow = datetime.datetime.utcnow()
             print "{0} {1}: ERROR    pid={2} died in initialization".format(str(timeNow),
                                                                             self.__class__.__name__,
                                                                             knight.pid)
             os.killpg(os.getpgrp(),signal.SIGKILL)
     # join
     for knight in knightList:    
         knight.join()
Ejemplo n.º 6
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

import multiprocessing

from pandajedi.jediorder import PostProcessor

parent_conn, child_conn = multiprocessing.Pipe()

postProcessor = multiprocessing.Process(target=PostProcessor.launcher,
                                        args=(child_conn,tbIF,ddmIF))
postProcessor.start()
Ejemplo n.º 7
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediorder.JobThrottler import JobThrottler

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

vo = 'atlas'
prodSourceLabel = 'managed'
cloud = 'WORLD'

# get SiteMapper
siteMapper = tbIF.getSiteMapper()
wqMap = tbIF.getWorkQueueMap()

jt = JobThrottler(vo, prodSourceLabel)
jt.initializeMods(tbIF)

workQueues = wqMap.getAlignedQueueList(vo, prodSourceLabel)
resource_types = tbIF.load_resource_types()
for workQueue in workQueues:
    for resource_type in resource_types:
        print(
            jt.toBeThrottled(vo, prodSourceLabel, cloud, workQueue,
                             resource_type.resource_name))
Ejemplo n.º 8
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

# Set up the task buffer interface
tb_if = JediTaskBufferInterface()
tb_if.setupInterface()

# Get the site mapper
site_mapper = tb_if.getSiteMapper()

# Define a random site of panda sites
panda_sites = ['ANALY_DESY-HH', 'ANALY_DESY-HH_TEST', 'WT2_Install', 'BNL_Test_2_CE_1', 'ANALY_IN2P3-CC-T2_RD',
               'UKI-SOUTHGRID-OX-HEP_SL6', 'RU-Protvino-IHEP', 'ANALY_NICS_Kraken', 'ANALY_TRIUMF_HIMEM',
               'TESTGLEXEC', 'ANALY_GRIF-LPNHE', 'UKI-SCOTGRID-GLASGOW_MCORE', 'BNL_PROD', 'RAL-LCG2_SL6']

# Test the getSiteInputStorageEndpointMap function
# print tb_if.getSiteInputStorageEndpointMap(panda_sites, site_mapper)

# for panda_site in panda_sites:
#     print panda_site
#     tmp_site_spec = site_mapper.getSite('ANALY_DESY-HH')
#     print '------------------- ddm -------------------'
#     print 'ddm_input: {0}, ddm_output: {1}'.format(tmp_site_spec.ddm_input, tmp_site_spec.ddm_output)
#     print '------------------- setokens values -------------------'
#     print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input.values(),
#                                                              tmp_site_spec.setokens_output.values())
#     print '------------------- setokens -------------------'
#     print 'setokens_input: {0}, setokens_output: {1}'.format(tmp_site_spec.setokens_input,
#                                                              tmp_site_spec.setokens_output)

from pandaserver.dataservice import DataServiceUtils    # noqa: E402
sites = site_mapper.getCloud('WORLD')['sites']
Ejemplo n.º 9
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()
wqMap = tbIF.getWorkQueueMap()
Ejemplo n.º 10
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()
atlasIF = ddmIF.getInterface('atlas')

from pandajedi.jedibrokerage.AtlasProdJobBroker import AtlasProdJobBroker
br = AtlasProdJobBroker(atlasIF,tbIF,siteMapper)

st,taskSpec = tbIF.getTaskWithID_JEDI(1)
print br.doBrokerage(taskSpec,'US',
  #{'mc12_8TeV.167751.Sherpa_CT10_ZeeMassiveCBPt0_CVetoBVeto.recon.AOD.e1585_a159_a171_tid01225039_00':['AOD.01225039._002187.pool.root.1','AOD.01225039._002627.pool.root.1']})
  {'data11_7TeV.00190256.physics_JetTauEtmiss.merge.NTUP_SMWZ.f408_m1007_p1035_tid01048572_00':['NTUP_SMWZ.01048572._000522.root.1']})
Ejemplo n.º 11
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jedicore.MsgWrapper import MsgWrapper

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jedirefine import RefinerUtils
from pandajedi.jediorder.TaskRefiner import TaskRefiner

from pandajedi.jediddm.DDMInterface import DDMInterface

import sys

logger = PandaLogger().getLogger('TaskRefiner')
tmpLog = MsgWrapper(logger)

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

ddmIF = DDMInterface()
ddmIF.setupInterface()

jediTaskID = int(sys.argv[1])

s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
refiner = TaskRefiner(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel)
refiner.initializeMods(tbIF, ddmIF)

taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID)
taskParamMap = RefinerUtils.decodeJSON(taskParam)
Ejemplo n.º 12
0
import sys

try:
    metaID = sys.argv[1]
except:
    metaID = None
import json
import uuid

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

reqIdx = 100000000
dsName = (
    "mc12_8TeV.129933.McAtNloJimmy_AUET2CT10_WW_e_e_0_0_0_0_0.merge.AOD.e1563_s1499_s1504_r3658_r3549_tid00999168_00"
)
logDatasetName = "panda.jeditest.log.{0}".format(uuid.uuid4())

taskParamMap = {}

taskParamMap["nFilesPerJob"] = 1
taskParamMap["nFiles"] = 2
taskParamMap["nEventsPerInputFile"] = 10000
taskParamMap["nEventsPerJob"] = 10000
# taskParamMap['nEvents'] = 25000
# taskParamMap['skipScout'] = True
taskParamMap["taskName"] = str(uuid.uuid4())
taskParamMap["userName"] = "******"
taskParamMap["vo"] = "atlas"
Ejemplo n.º 13
0
    def start(self):
        # start zombi cleaner
        ZombiCleaner().start()
        # setup DDM I/F
        ddmIF = DDMInterface()
        ddmIF.setupInterface()
        # setup TaskBuffer I/F
        taskBufferIF = JediTaskBufferInterface()
        taskBufferIF.setupInterface()
        # setup intra-node message queue broker proxies
        if hasattr(jedi_config, 'mq') and hasattr(
                jedi_config.mq, 'configFile') and jedi_config.mq.configFile:
            from pandajedi.jediorder.JediMsgProcessor import MsgProcAgent
            mq_agent = MsgProcAgent(config_file=jedi_config.mq.configFile)
            mb_proxy_dict = mq_agent.start_passive_mode(prefetch_size=999)
        # the list of JEDI knights
        knightList = []
        # setup TaskRefiner
        for itemStr in jedi_config.taskrefine.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskRefiner', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup TaskBrokerage
        for itemStr in jedi_config.taskbroker.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskBroker', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup ContentsFeeder
        for itemStr in jedi_config.confeeder.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.ContentsFeeder', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup JobGenerator
        for itemStr in jedi_config.jobgen.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            cloud = items[3]
            try:
                loop_cycle = items[4]
            except IndexError:
                loop_cycle = None

            if not isinstance(cloud, list):
                cloud = [cloud]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = ProcessWrapper(target=self.launcher,
                                      args=('pandajedi.jediorder.JobGenerator',
                                            child_conn, taskBufferIF, ddmIF,
                                            vo, plabel, cloud, True, True,
                                            loop_cycle))
                proc.start()
                knightList.append(proc)
        # setup PostProcessor
        for itemStr in jedi_config.postprocessor.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.PostProcessor', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup TaskCommando
        for itemStr in jedi_config.tcommando.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.TaskCommando', child_conn,
                          taskBufferIF, ddmIF, vo, plabel))
                proc.start()
                knightList.append(proc)
        # setup WatchDog
        for itemStr in jedi_config.watchdog.procConfig.split(';'):
            items = self.convParams(itemStr)
            vo = items[0]
            plabel = items[1]
            nProc = items[2]
            subStr = items[3] if len(items) > 3 else None
            period = items[4] if len(items) > 4 else None
            for iproc in range(nProc):
                parent_conn, child_conn = multiprocessing.Pipe()
                proc = multiprocessing.Process(
                    target=self.launcher,
                    args=('pandajedi.jediorder.WatchDog', child_conn,
                          taskBufferIF, ddmIF, vo, plabel, subStr, period))
                proc.start()
                knightList.append(proc)
        # setup JediMsgProcessor agent (only one system process)
        if hasattr(jedi_config, 'msgprocessor') and hasattr(
                jedi_config.msgprocessor,
                'configFile') and jedi_config.msgprocessor.configFile:
            stop_event = multiprocessing.Event()
            self.stopEventList.append(stop_event)
            parent_conn, child_conn = multiprocessing.Pipe()
            proc = multiprocessing.Process(
                target=self.launcher,
                args=('pandajedi.jediorder.JediMsgProcessor', stop_event))
            proc.start()
            knightList.append(proc)
        # setup JediDaemon agent (only one system process)
        if hasattr(jedi_config, 'daemon') and hasattr(
                jedi_config.daemon, 'enable') and jedi_config.daemon.enable:
            parent_conn, child_conn = multiprocessing.Pipe()
            proc = multiprocessing.Process(
                target=self.launcher,
                args=('pandajedi.jediorder.JediDaemon', taskBufferIF, ddmIF))
            proc.start()
            knightList.append(proc)
        # check initial failures
        time.sleep(5)
        for knight in knightList:
            if not knight.is_alive():
                timeNow = datetime.datetime.utcnow()
                print(
                    "{0} {1}: ERROR    pid={2} died in initialization".format(
                        str(timeNow), self.__class__.__name__, knight.pid))
                os.killpg(os.getpgrp(), signal.SIGKILL)
        # join
        for knight in knightList:
            knight.join()
Ejemplo n.º 14
0
# logger
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface

from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import os
import sys
import socket

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

ddmIF = DDMInterface()
ddmIF.setupInterface()

jediTaskID = int(sys.argv[1])

# get task attributes
s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
pid = '{0}-{1}_{2}-sgen'.format(socket.getfqdn().split('.')[0], os.getpid(),
                                os.getpgrp())
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = tbIF.getWorkQueueMap().getQueueWithIDGshare(
    taskSpec.workQueue_ID, taskSpec.gshare)
Ejemplo n.º 15
0
from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

from pandajedi.jediddm.DDMInterface import DDMInterface

from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys

logger = PandaLogger().getLogger('JobGenerator')
tmpLog = MsgWrapper(logger)

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

ddmIF = DDMInterface()
ddmIF.setupInterface()

jediTaskID = int(sys.argv[1])

datasetIDs = None
if len(sys.argv) > 2:
    datasetIDs = [int(sys.argv[2])]

s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID)
Ejemplo n.º 16
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
from pandajedi.jedicore.MsgWrapper import MsgWrapper

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jedirefine import RefinerUtils
from pandajedi.jediorder.TaskBroker import TaskBroker

from pandajedi.jediddm.DDMInterface import DDMInterface

import sys

logger = PandaLogger().getLogger('TaskBroker')
tmpLog = MsgWrapper(logger)

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

ddmIF = DDMInterface()
ddmIF.setupInterface()

jediTaskID = int(sys.argv[1])

s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID, False)
body = TaskBroker(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel)
body.initializeMods(tbIF, ddmIF)

taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID)
taskParamMap = RefinerUtils.decodeJSON(taskParam)
Ejemplo n.º 17
0
import sys
import uuid

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

from pandajedi.jedicore.JediTaskSpec import JediTaskSpec

task = JediTaskSpec()
task.jediTaskID = sys.argv[1]
task.taskName = 'pandatest.{0}'.format(uuid.uuid4())
task.status = 'defined'
task.userName = '******'
task.vo = 'atlas'
task.prodSourceLabel = 'managed'
task.taskPriority = 100
task.currentPriority = task.taskPriority
task.architecture = 'i686-slc5-gcc43-opt'
task.transUses = 'Atlas-17.2.7'
task.transHome = 'AtlasProduction-17.2.8.10'
task.transPath = 'Reco_trf.py'
task.workQueue_ID = 3

tbIF.insertTask_JEDI(task)

from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec

ds = JediDatasetSpec()
ds.jediTaskID = task.jediTaskID
Ejemplo n.º 18
0
import sys
try:
    metaID = sys.argv[1]
except:
    metaID = None
import json
import uuid

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

inFileList = ['file1','file2','file3']

logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4())

taskParamMap = {}

taskParamMap['nFilesPerJob'] = 1
taskParamMap['nFiles'] = len(inFileList)
#taskParamMap['nEventsPerInputFile']  = 10000
#taskParamMap['nEventsPerJob'] = 10000
#taskParamMap['nEvents'] = 25000
taskParamMap['noInput'] = True
taskParamMap['pfnList'] = inFileList
#taskParamMap['mergeOutput'] = True
taskParamMap['taskName'] = str(uuid.uuid4())
taskParamMap['userName'] = '******'
taskParamMap['vo'] = 'nonatlas'
taskParamMap['taskPriority'] = 900
Ejemplo n.º 19
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool,ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys
jediTaskID = int(sys.argv[1])
try:
    datasetID = [int(sys.argv[2])]
except:
    datasetID = None
Ejemplo n.º 20
0
# logger
from pandacommon.pandalogger.PandaLogger import PandaLogger
logger = PandaLogger().getLogger('JobGenerator')
from pandajedi.jedicore.MsgWrapper import MsgWrapper
tmpLog = MsgWrapper(logger)

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()

siteMapper = tbIF.getSiteMapper()

from pandajedi.jediddm.DDMInterface import DDMInterface

ddmIF = DDMInterface()
ddmIF.setupInterface()

from pandajedi.jediorder.JobBroker import JobBroker
from pandajedi.jediorder.JobSplitter import JobSplitter
from pandajedi.jediorder.JobGenerator import JobGeneratorThread
from pandajedi.jedicore.ThreadUtils import ThreadPool, ListWithLock
from pandajedi.jediorder.TaskSetupper import TaskSetupper

import sys
jediTaskID = int(sys.argv[1])
try:
    datasetID = [int(sys.argv[2])]
except:
    datasetID = None
Ejemplo n.º 21
0
import sys
try:
    testTaskType = sys.argv[1]
except Exception:
    testTaskType = 'test'

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface
from pandajedi.jediddm.DDMInterface import DDMInterface

import multiprocessing

from pandajedi.jediorder import WatchDog

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()


ddmIF = DDMInterface()
ddmIF.setupInterface()


parent_conn, child_conn = multiprocessing.Pipe()

watchDog = multiprocessing.Process(target=WatchDog.launcher,
                                   args=(child_conn,tbIF,ddmIF,
                                         'atlas',testTaskType))
watchDog.start()
Ejemplo n.º 22
0
import sys
import uuid

from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface

tbIF = JediTaskBufferInterface()
tbIF.setupInterface()


from pandajedi.jedicore.JediTaskSpec import JediTaskSpec

task = JediTaskSpec()
task.jediTaskID = sys.argv[1]
task.taskName = 'pandatest.{0}'.format(uuid.uuid4())
task.status = 'defined'
task.userName = '******'
task.vo = 'atlas'
task.prodSourceLabel = 'managed'
task.taskPriority = 100
task.currentPriority = task.taskPriority
task.architecture = 'i686-slc5-gcc43-opt'
task.transUses = 'Atlas-17.2.7'
task.transHome = 'AtlasProduction-17.2.8.10'
task.transPath = 'Reco_trf.py'
task.workQueue_ID = 3

tbIF.insertTask_JEDI(task) 

from pandajedi.jedicore.JediDatasetSpec import JediDatasetSpec
ds = JediDatasetSpec()
ds.jediTaskID = task.jediTaskID