def run(dn, jobID, cloud=None, excludedSite=None): # check parameters if dn == '': return False if jobID < 0: return False # password from config import panda_config passwd = panda_config.dbpasswd # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB from taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # run ReBroker from userinterface.ReBroker import ReBroker reThr = ReBroker(taskBuffer, cloud, excludedSite, userRequest=True) # lock stLock, retLock = reThr.lockJob(dn, jobID) # failed if not stLock: return False # start reThr.start() reThr.join() return True
def run(inFile,v_onlyTA): import cPickle as pickle try: # read Jobs from file f = open(inFile) jobs = pickle.load(f) f.close() except: type, value, traceBack = sys.exc_info() print("run() : %s %s" % (type,value)) return # password from config import panda_config passwd = panda_config.dbpasswd # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB from taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # run Setupper from dataservice.Setupper import Setupper thr = Setupper(taskBuffer,jobs,onlyTA=v_onlyTA) thr.start() thr.join() return
def run(inFile,v_onlyTA,v_firstSubmission): import cPickle as pickle try: # read Jobs from file f = open(inFile) jobs = pickle.load(f) f.close() except: type, value, traceBack = sys.exc_info() print("run() : %s %s" % (type,value)) return # password from config import panda_config passwd = panda_config.dbpasswd # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB from taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # run Setupper from dataservice.Setupper import Setupper thr = Setupper(taskBuffer,jobs,onlyTA=v_onlyTA,firstSubmission=v_firstSubmission) thr.start() thr.join() return
def main(backGround=False): _logger.debug('starting ...') # register signal handler signal.signal(signal.SIGINT, catch_sig) signal.signal(signal.SIGHUP, catch_sig) signal.signal(signal.SIGTERM, catch_sig) signal.signal(signal.SIGALRM, catch_sig) signal.alarm(overallTimeout) # forking pid = os.fork() if pid != 0: # watch child process os.wait() time.sleep(1) else: # main loop from taskbuffer.TaskBuffer import taskBuffer # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # ActiveMQ params clientid = 'PANDA-' + socket.getfqdn() queue = '/queue/Consumer.test1.poc.pocMSG' ssl_opts = { 'use_ssl': True, 'ssl_cert_file': '%s/hostcert.pem' % panda_config.certdir, 'ssl_key_file': '%s/hostkey.pem' % panda_config.certdir } # resolve multiple brokers brokerList = socket.gethostbyname_ex('gridmsg007.cern.ch')[-1] # set listener for tmpBroker in brokerList: try: _logger.debug('setting listener on %s' % tmpBroker) conn = stomp.Connection(host_and_ports=[(tmpBroker, 6162)], **ssl_opts) conn.set_listener( 'GenCallbackConsumer', GenCallbackConsumer(conn, taskBuffer, siteMapper)) conn.start() conn.connect(headers={'client-id': clientid}) conn.subscribe(destination=queue, ack='client-individual') #,headers = {'selector':"cbtype='FileDoneMessage'"}) if not conn.is_connected(): _logger.error("connection failure to %s" % tmpBroker) except: errtype, errvalue = sys.exc_info()[:2] _logger.error("failed to set listener on %s : %s %s" % (tmpBroker, errtype, errvalue)) catch_sig(None, None)
def main(backGround=False): _logger.debug('starting ...') # register signal handler signal.signal(signal.SIGINT, catch_sig) signal.signal(signal.SIGHUP, catch_sig) signal.signal(signal.SIGTERM,catch_sig) signal.signal(signal.SIGALRM,catch_sig) signal.alarm(overallTimeout) # forking pid = os.fork() if pid != 0: # watch child process os.wait() time.sleep(1) else: # main loop from taskbuffer.TaskBuffer import taskBuffer # check certificate certName = '/data/atlpan/pandasv1_usercert.pem' _logger.debug('checking certificate {0}'.format(certName)) certOK,certMsg = DataServiceUtils.checkCertificate(certName) if not certOK: _logger.error('bad certificate : {0}'.format(certMsg)) # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # ActiveMQ params queue = '/queue/Consumer.PANDA.atlas.ddm.siteservices' ssl_opts = {'use_ssl' : True, 'ssl_cert_file' : certName, 'ssl_key_file' : '/data/atlpan/pandasv1_userkey.pem'} # resolve multiple brokers brokerList = socket.gethostbyname_ex('atlasddm-mb.cern.ch')[-1] # set listener for tmpBroker in brokerList: try: clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker _logger.debug('setting listener %s' % clientid) conn = stomp.Connection(host_and_ports = [(tmpBroker, 6162)], **ssl_opts) conn.set_listener('FileCallbackListener', FileCallbackListener(conn,taskBuffer,siteMapper)) conn.start() conn.connect(headers = {'client-id': clientid}) conn.subscribe(destination=queue, ack='client-individual') #,headers = {'selector':"cbtype='FileDoneMessage'"}) if not conn.is_connected(): _logger.error("connection failure to %s" % tmpBroker) _logger.debug('listener %s is up and running' % clientid) except: errtype,errvalue = sys.exc_info()[:2] _logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue)) catch_sig(None,None)
def main(backGround=False): _logger.debug('starting ...') # register signal handler signal.signal(signal.SIGINT, catch_sig) signal.signal(signal.SIGHUP, catch_sig) signal.signal(signal.SIGTERM,catch_sig) signal.signal(signal.SIGALRM,catch_sig) signal.alarm(overallTimeout) # forking pid = os.fork() if pid != 0: # watch child process os.wait() time.sleep(1) else: # main loop from taskbuffer.TaskBuffer import taskBuffer # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # ActiveMQ params clientid = 'PANDA-' + socket.getfqdn() queue = '/queue/Consumer.test1.poc.pocMSG' ssl_opts = {'use_ssl' : True, 'ssl_cert_file' : '%s/hostcert.pem'%panda_config.certdir, 'ssl_key_file' : '%s/hostkey.pem'%panda_config.certdir} # resolve multiple brokers brokerList = socket.gethostbyname_ex('gridmsg007.cern.ch')[-1] # set listener for tmpBroker in brokerList: try: _logger.debug('setting listener on %s' % tmpBroker) conn = stomp.Connection(host_and_ports = [(tmpBroker, 6162)], **ssl_opts) conn.set_listener('GenCallbackConsumer', GenCallbackConsumer(conn,taskBuffer,siteMapper)) conn.start() conn.connect(headers = {'client-id': clientid}) conn.subscribe(destination=queue, ack='client-individual') #,headers = {'selector':"cbtype='FileDoneMessage'"}) if not conn.is_connected(): _logger.error("connection failure to %s" % tmpBroker) except: errtype,errvalue = sys.exc_info()[:2] _logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue)) catch_sig(None,None)
#!/usr/bin/python """ entry point """ import datetime import traceback import types # config file from config import panda_config # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # initialzie TaskBuffer from taskbuffer.TaskBuffer import taskBuffer taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, panda_config.nDBConnection, True) # initialize JobDispatcher from jobdispatcher.JobDispatcher import jobDispatcher if panda_config.nDBConnection != 0: jobDispatcher.init(taskBuffer) # initialize DataService from dataservice.DataService import dataService if panda_config.nDBConnection != 0: dataService.init(taskBuffer)
import os import re import sys import time import datetime import commands import threading from pandajedi.jediconfig import jedi_config from config import panda_config # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() from pandajedi.jedicore import JediTaskBuffer from pandalogger.PandaLogger import PandaLogger taskBuffer= JediTaskBuffer.JediTaskBuffer(None) proxy = taskBuffer.proxyPool.getProxy() proxy.refreshWrokQueueMap() """ print proxy.workQueueMap.dump() print proxy.workQueueMap.getQueueWithSelParams('atlas','managed', prodSourceLabel='managed', workingGroup='GP_Top')[0].queue_name print proxy.workQueueMap.getQueueWithSelParams('atlas','managed', prodSourceLabel='managed', workingGroup='AP_Top',
def main(backGround=False): _logger.debug('starting ...') # register signal handler signal.signal(signal.SIGINT, catch_sig) signal.signal(signal.SIGHUP, catch_sig) signal.signal(signal.SIGTERM, catch_sig) signal.signal(signal.SIGALRM, catch_sig) signal.alarm(overallTimeout) # forking pid = os.fork() if pid != 0: # watch child process os.wait() time.sleep(1) else: # main loop from taskbuffer.TaskBuffer import taskBuffer # check certificate certName = '%s/pandasv1_usercert.pem' % panda_config.certdir keyName = '%s/pandasv1_userkey.pem' % panda_config.certdir _logger.debug('checking certificate {0}'.format(certName)) certOK, certMsg = DataServiceUtils.checkCertificate(certName) if not certOK: _logger.error('bad certificate : {0}'.format(certMsg)) # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # ActiveMQ params queue = '/queue/Consumer.PANDA.atlas.ddm.siteservices' ssl_opts = { 'use_ssl': True, 'ssl_version': ssl.PROTOCOL_TLSv1, 'ssl_cert_file': certName, 'ssl_key_file': keyName } # resolve multiple brokers brokerList = socket.gethostbyname_ex('atlas-mb.cern.ch')[-1] # set listener connList = [] for tmpBroker in brokerList: try: clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker subscription_id = 'panda-server-consumer-' + socket.getfqdn() _logger.debug('setting listener %s' % clientid) conn = stomp.Connection(host_and_ports=[(tmpBroker, 61023)], **ssl_opts) connList.append(conn) except: errtype, errvalue = sys.exc_info()[:2] _logger.error("failed to connect to %s : %s %s" % (tmpBroker, errtype, errvalue)) catch_sig(None, None) while True: for conn in connList: try: if not conn.is_connected(): conn.set_listener( 'FileCallbackListener', FileCallbackListener(conn, taskBuffer, siteMapper, subscription_id)) conn.start() conn.connect(headers={'client-id': clientid}) conn.subscribe(destination=queue, id=subscription_id, ack='client-individual') _logger.debug('listener %s is up and running' % clientid) except: errtype, errvalue = sys.exc_info()[:2] _logger.error("failed to set listener on %s : %s %s" % (tmpBroker, errtype, errvalue)) catch_sig(None, None) time.sleep(5)
def main(backGround=False): _logger.debug('starting ...') # register signal handler signal.signal(signal.SIGINT, catch_sig) signal.signal(signal.SIGHUP, catch_sig) signal.signal(signal.SIGTERM,catch_sig) signal.signal(signal.SIGALRM,catch_sig) signal.alarm(overallTimeout) # forking pid = os.fork() if pid != 0: # watch child process os.wait() time.sleep(1) else: # main loop from taskbuffer.TaskBuffer import taskBuffer # check certificate certName = '/data/atlpan/pandasv1_usercert.pem' #certName = '/etc/grid-security/hostcert.pem' _logger.debug('checking certificate {0}'.format(certName)) certOK,certMsg = DataServiceUtils.checkCertificate(certName) if not certOK: _logger.error('bad certificate : {0}'.format(certMsg)) # initialize cx_Oracle using dummy connection from taskbuffer.Initializer import initializer initializer.init() # instantiate TB taskBuffer.init(panda_config.dbhost,panda_config.dbpasswd,nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # ActiveMQ params queue = '/queue/Consumer.panda.rucio.events' ssl_opts = {'use_ssl' : True, 'ssl_version' : ssl.PROTOCOL_TLSv1, 'ssl_cert_file' : certName, 'ssl_key_file' : '/data/atlpan/pandasv1_userkey.pem'} # resolve multiple brokers brokerList = socket.gethostbyname_ex('atlas-mb.cern.ch')[-1] # set listener connList = [] for tmpBroker in brokerList: try: clientid = 'PANDA-' + socket.getfqdn() + '-' + tmpBroker subscription_id = 'panda-server-consumer' _logger.debug('setting listener %s to broker %s' % (clientid, tmpBroker)) conn = stomp.Connection(host_and_ports = [(tmpBroker, 61023)], **ssl_opts) connList.append(conn) except: errtype,errvalue = sys.exc_info()[:2] _logger.error("failed to connect to %s : %s %s" % (tmpBroker,errtype,errvalue)) catch_sig(None,None) while True: for conn in connList: try: if not conn.is_connected(): conn.set_listener('DatasetCallbackListener', DatasetCallbackListener(conn,taskBuffer,siteMapper, subscription_id)) conn.start() conn.connect(headers = {'client-id': clientid}) conn.subscribe(destination=queue, id=subscription_id, ack='auto') _logger.debug('listener %s is up and running' % clientid) except: errtype,errvalue = sys.exc_info()[:2] _logger.error("failed to set listener on %s : %s %s" % (tmpBroker,errtype,errvalue)) catch_sig(None,None) time.sleep(5)