def run(dn, jobID, cloud=None, excludedSite=None):
# check parameters
if dn == '':
return False
if jobID < 0:
return False
# password
from config import panda_config
passwd = panda_config.dbpasswd
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
from taskbuffer.TaskBuffer import taskBuffer
taskBuffer.init(panda_config.dbhost,
panda_config.dbpasswd,
nDBConnection=1)
# run ReBroker
from userinterface.ReBroker import ReBroker
reThr = ReBroker(taskBuffer, cloud, excludedSite, userRequest=True)
# lock
stLock, retLock = reThr.lockJob(dn, jobID)
# failed
if not stLock:
return False
# start
reThr.start()
reThr.join()
return True
def run(inFile,v_onlyTA):
import cPickle as pickle
try:
# read Jobs from file
f = open(inFile)
jobs = pickle.load(f)
f.close()
except:
type, value, traceBack = sys.exc_info()
print("run() : %s %s" % (type,value))
return
# password
from config import panda_config
passwd = panda_config.dbpasswd
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
from taskbuffer.TaskBuffer import taskBuffer
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# run Setupper
from dataservice.Setupper import Setupper
thr = Setupper(taskBuffer,jobs,onlyTA=v_onlyTA)
thr.start()
thr.join()
return
def run(inFile,v_onlyTA,v_firstSubmission):
import cPickle as pickle
try:
# read Jobs from file
f = open(inFile)
jobs = pickle.load(f)
f.close()
except:
type, value, traceBack = sys.exc_info()
print("run() : %s %s" % (type,value))
return
# password
from config import panda_config
passwd = panda_config.dbpasswd
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
from taskbuffer.TaskBuffer import taskBuffer
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# run Setupper
from dataservice.Setupper import Setupper
thr = Setupper(taskBuffer,jobs,onlyTA=v_onlyTA,firstSubmission=v_firstSubmission)
thr.start()
thr.join()
return
def main(backGround=False):
_logger.debug('starting ...')
# register signal handler
signal.signal(signal.SIGINT, catch_sig)
signal.signal(signal.SIGHUP, catch_sig)
signal.signal(signal.SIGTERM, catch_sig)
signal.signal(signal.SIGALRM, catch_sig)
signal.alarm(overallTimeout)
# forking
pid = os.fork()
if pid != 0:
# watch child process
os.wait()
time.sleep(1)
else:
# main loop
from taskbuffer.TaskBuffer import taskBuffer
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
taskBuffer.init(panda_config.dbhost,
panda_config.dbpasswd,
nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
# ActiveMQ params
clientid = 'PANDA-' + socket.getfqdn()
queue = '/queue/Consumer.test1.poc.pocMSG'
ssl_opts = {
'use_ssl': True,
'ssl_cert_file': '%s/hostcert.pem' % panda_config.certdir,
'ssl_key_file': '%s/hostkey.pem' % panda_config.certdir
}
# resolve multiple brokers
brokerList = socket.gethostbyname_ex('gridmsg007.cern.ch')[-1]
# set listener
for tmpBroker in brokerList:
try:
_logger.debug('setting listener on %s' % tmpBroker)
conn = stomp.Connection(host_and_ports=[(tmpBroker, 6162)],
**ssl_opts)
conn.set_listener(
'GenCallbackConsumer',
GenCallbackConsumer(conn, taskBuffer, siteMapper))
conn.start()
conn.connect(headers={'client-id': clientid})
conn.subscribe(destination=queue, ack='client-individual')
#,headers = {'selector':"cbtype='FileDoneMessage'"})
if not conn.is_connected():
_logger.error("connection failure to %s" % tmpBroker)
except:
errtype, errvalue = sys.exc_info()[:2]
_logger.error("failed to set listener on %s : %s %s" %
(tmpBroker, errtype, errvalue))
catch_sig(None, None)
def main(backGround=False):
_logger.debug('starting ...')
# register signal handler
signal.signal(signal.SIGINT, catch_sig)
signal.signal(signal.SIGHUP, catch_sig)
signal.signal(signal.SIGTERM,catch_sig)
signal.signal(signal.SIGALRM,catch_sig)
signal.alarm(overallTimeout)
# forking
pid = os.fork()
if pid != 0:
# watch child process
os.wait()
time.sleep(1)
else:
# main loop
from taskbuffer.TaskBuffer import taskBuffer
# check certificate
certName = '/data/atlpan/pandasv1_usercert.pem'
_logger.debug('checking certificate {0}'.format(certName))
certOK,certMsg = DataServiceUtils.checkCertificate(certName)
if not certOK:
_logger.error('bad certificate : {0}'.format(certMsg))
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
# ActiveMQ params
queue = '/queue/Consumer.PANDA.atlas.ddm.siteservices'
ssl_opts = {'use_ssl' : True,
'ssl_cert_file' : certName,
'ssl_key_file' : '/data/atlpan/pandasv1_userkey.pem'}
# resolve multiple brokers
brokerList = socket.gethostbyname_ex('atlasddm-mb.cern.ch')[-1]
# set listener
for tmpBroker in brokerList:
try:
clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker
_logger.debug('setting listener %s' % clientid)
conn = stomp.Connection(host_and_ports = [(tmpBroker, 6162)], **ssl_opts)
conn.set_listener('FileCallbackListener', FileCallbackListener(conn,taskBuffer,siteMapper))
conn.start()
conn.connect(headers = {'client-id': clientid})
conn.subscribe(destination=queue, ack='client-individual')
#,headers = {'selector':"cbtype='FileDoneMessage'"})
if not conn.is_connected():
_logger.error("connection failure to %s" % tmpBroker)
_logger.debug('listener %s is up and running' % clientid)
except:
errtype,errvalue = sys.exc_info()[:2]
_logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue))
catch_sig(None,None)
def main(backGround=False):
_logger.debug('starting ...')
# register signal handler
signal.signal(signal.SIGINT, catch_sig)
signal.signal(signal.SIGHUP, catch_sig)
signal.signal(signal.SIGTERM,catch_sig)
signal.signal(signal.SIGALRM,catch_sig)
signal.alarm(overallTimeout)
# forking
pid = os.fork()
if pid != 0:
# watch child process
os.wait()
time.sleep(1)
else:
# main loop
from taskbuffer.TaskBuffer import taskBuffer
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
# ActiveMQ params
clientid = 'PANDA-' + socket.getfqdn()
queue = '/queue/Consumer.test1.poc.pocMSG'
ssl_opts = {'use_ssl' : True,
'ssl_cert_file' : '%s/hostcert.pem'%panda_config.certdir,
'ssl_key_file' : '%s/hostkey.pem'%panda_config.certdir}
# resolve multiple brokers
brokerList = socket.gethostbyname_ex('gridmsg007.cern.ch')[-1]
# set listener
for tmpBroker in brokerList:
try:
_logger.debug('setting listener on %s' % tmpBroker)
conn = stomp.Connection(host_and_ports = [(tmpBroker, 6162)], **ssl_opts)
conn.set_listener('GenCallbackConsumer', GenCallbackConsumer(conn,taskBuffer,siteMapper))
conn.start()
conn.connect(headers = {'client-id': clientid})
conn.subscribe(destination=queue, ack='client-individual')
#,headers = {'selector':"cbtype='FileDoneMessage'"})
if not conn.is_connected():
_logger.error("connection failure to %s" % tmpBroker)
except:
errtype,errvalue = sys.exc_info()[:2]
_logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue))
catch_sig(None,None)
#!/usr/bin/python
"""
entry point
"""
import datetime
import traceback
import types
# config file
from config import panda_config
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# initialzie TaskBuffer
from taskbuffer.TaskBuffer import taskBuffer
taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd,
panda_config.nDBConnection, True)
# initialize JobDispatcher
from jobdispatcher.JobDispatcher import jobDispatcher
if panda_config.nDBConnection != 0:
jobDispatcher.init(taskBuffer)
# initialize DataService
from dataservice.DataService import dataService
if panda_config.nDBConnection != 0:
dataService.init(taskBuffer)
import os
import re
import sys
import time
import datetime
import commands
import threading
from pandajedi.jediconfig import jedi_config
from config import panda_config
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
from pandajedi.jedicore import JediTaskBuffer
from pandalogger.PandaLogger import PandaLogger
taskBuffer= JediTaskBuffer.JediTaskBuffer(None)
proxy = taskBuffer.proxyPool.getProxy()
proxy.refreshWrokQueueMap()
"""
print proxy.workQueueMap.dump()
print proxy.workQueueMap.getQueueWithSelParams('atlas','managed',
prodSourceLabel='managed',
workingGroup='GP_Top')[0].queue_name
print proxy.workQueueMap.getQueueWithSelParams('atlas','managed',
prodSourceLabel='managed',
workingGroup='AP_Top',
def main(backGround=False):
_logger.debug('starting ...')
# register signal handler
signal.signal(signal.SIGINT, catch_sig)
signal.signal(signal.SIGHUP, catch_sig)
signal.signal(signal.SIGTERM, catch_sig)
signal.signal(signal.SIGALRM, catch_sig)
signal.alarm(overallTimeout)
# forking
pid = os.fork()
if pid != 0:
# watch child process
os.wait()
time.sleep(1)
else:
# main loop
from taskbuffer.TaskBuffer import taskBuffer
# check certificate
certName = '%s/pandasv1_usercert.pem' % panda_config.certdir
keyName = '%s/pandasv1_userkey.pem' % panda_config.certdir
_logger.debug('checking certificate {0}'.format(certName))
certOK, certMsg = DataServiceUtils.checkCertificate(certName)
if not certOK:
_logger.error('bad certificate : {0}'.format(certMsg))
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
taskBuffer.init(panda_config.dbhost,
panda_config.dbpasswd,
nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
# ActiveMQ params
queue = '/queue/Consumer.PANDA.atlas.ddm.siteservices'
ssl_opts = {
'use_ssl': True,
'ssl_version': ssl.PROTOCOL_TLSv1,
'ssl_cert_file': certName,
'ssl_key_file': keyName
}
# resolve multiple brokers
brokerList = socket.gethostbyname_ex('atlas-mb.cern.ch')[-1]
# set listener
connList = []
for tmpBroker in brokerList:
try:
clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker
subscription_id = 'panda-server-consumer-' + socket.getfqdn()
_logger.debug('setting listener %s' % clientid)
conn = stomp.Connection(host_and_ports=[(tmpBroker, 61023)],
**ssl_opts)
connList.append(conn)
except:
errtype, errvalue = sys.exc_info()[:2]
_logger.error("failed to connect to %s : %s %s" %
(tmpBroker, errtype, errvalue))
catch_sig(None, None)
while True:
for conn in connList:
try:
if not conn.is_connected():
conn.set_listener(
'FileCallbackListener',
FileCallbackListener(conn, taskBuffer, siteMapper,
subscription_id))
conn.start()
conn.connect(headers={'client-id': clientid})
conn.subscribe(destination=queue,
id=subscription_id,
ack='client-individual')
_logger.debug('listener %s is up and running' %
clientid)
except:
errtype, errvalue = sys.exc_info()[:2]
_logger.error("failed to set listener on %s : %s %s" %
(tmpBroker, errtype, errvalue))
catch_sig(None, None)
time.sleep(5)
def main(backGround=False):
_logger.debug('starting ...')
# register signal handler
signal.signal(signal.SIGINT, catch_sig)
signal.signal(signal.SIGHUP, catch_sig)
signal.signal(signal.SIGTERM,catch_sig)
signal.signal(signal.SIGALRM,catch_sig)
signal.alarm(overallTimeout)
# forking
pid = os.fork()
if pid != 0:
# watch child process
os.wait()
time.sleep(1)
else:
# main loop
from taskbuffer.TaskBuffer import taskBuffer
# check certificate
certName = '/data/atlpan/pandasv1_usercert.pem'
#certName = '/etc/grid-security/hostcert.pem'
_logger.debug('checking certificate {0}'.format(certName))
certOK,certMsg = DataServiceUtils.checkCertificate(certName)
if not certOK:
_logger.error('bad certificate : {0}'.format(certMsg))
# initialize cx_Oracle using dummy connection
from taskbuffer.Initializer import initializer
initializer.init()
# instantiate TB
taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1)
# instantiate sitemapper
siteMapper = SiteMapper(taskBuffer)
# ActiveMQ params
queue = '/queue/Consumer.panda.rucio.events'
ssl_opts = {'use_ssl' : True,
'ssl_version' : ssl.PROTOCOL_TLSv1,
'ssl_cert_file' : certName,
'ssl_key_file' : '/data/atlpan/pandasv1_userkey.pem'}
# resolve multiple brokers
brokerList = socket.gethostbyname_ex('atlas-mb.cern.ch')[-1]
# set listener
connList = []
for tmpBroker in brokerList:
try:
clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker
subscription_id = 'panda-server-consumer'
_logger.debug('setting listener %s to broker %s' % (clientid, tmpBroker))
conn = stomp.Connection(host_and_ports = [(tmpBroker, 61023)], **ssl_opts)
connList.append(conn)
except:
errtype,errvalue = sys.exc_info()[:2]
_logger.error("failed to connect to %s : %s %s" % (tmpBroker,errtype,errvalue))
catch_sig(None,None)
while True:
for conn in connList:
try:
if not conn.is_connected():
conn.set_listener('DatasetCallbackListener', DatasetCallbackListener(conn,taskBuffer,siteMapper,
subscription_id))
conn.start()
conn.connect(headers = {'client-id': clientid})
conn.subscribe(destination=queue, id=subscription_id, ack='auto')
_logger.debug('listener %s is up and running' % clientid)
except:
errtype,errvalue = sys.exc_info()[:2]
_logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue))
catch_sig(None,None)
time.sleep(5)