def bulkSendMsg(self, msgType, msgLevel='info'): try: nChunk = 20 for iMsg, message in enumerate(self.msgBuffer): # get logger tmpPandaLogger = PandaLogger() # lock HTTP handler tmpPandaLogger.lock() tmpPandaLogger.setParams({'Type': msgType}) # get logger tmpLogger = tmpPandaLogger.getHttpLogger( jedi_config.master.loggername) # add message message = self.monToken + ' ' + message if msgLevel == 'error': tmpLogger.error(message) elif msgLevel == 'warning': tmpLogger.warning(message) elif msgLevel == 'info': tmpLogger.info(message) else: tmpLogger.debug(message) # release HTTP handler tmpPandaLogger.release() if (iMsg + 1) % nChunk == 0: time.sleep(1) except: pass
def sendMsg(self, message, msgType, msgLevel='info', escapeChar=False): try: # get logger tmpPandaLogger = PandaLogger() # lock HTTP handler tmpPandaLogger.lock() tmpPandaLogger.setParams({'Type': msgType}) # get logger tmpLogger = tmpPandaLogger.getHttpLogger( jedi_config.master.loggername) # escape special characters if escapeChar: message = message.replace('<', '<') message = message.replace('>', '>') # add message message = self.monToken + ' ' + message if msgLevel == 'error': tmpLogger.error(message) elif msgLevel == 'warning': tmpLogger.warning(message) elif msgLevel == 'info': tmpLogger.info(message) else: tmpLogger.debug(message) # release HTTP handler tmpPandaLogger.release() except: pass
def bulkSendMsg(self,msgType,msgLevel='info',loggerName=None): try: nChunk = 20 if loggerName == None: loggerName = jedi_config.master.loggername for iMsg,message in enumerate(self.msgBuffer): # get logger tmpPandaLogger = PandaLogger() # lock HTTP handler tmpPandaLogger.lock() tmpPandaLogger.setParams({'Type':msgType}) # get logger tmpLogger = tmpPandaLogger.getHttpLogger(loggerName) # add message message = self.monToken + ' ' + message if msgLevel=='error': tmpLogger.error(message) elif msgLevel=='warning': tmpLogger.warning(message) elif msgLevel=='info': tmpLogger.info(message) else: tmpLogger.debug(message) # release HTTP handler tmpPandaLogger.release() if (iMsg+1) % nChunk == 0: time.sleep(1) except: pass
def sendMsg(self,message,msgType,msgLevel='info',escapeChar=False): try: # get logger tmpPandaLogger = PandaLogger() # lock HTTP handler tmpPandaLogger.lock() tmpPandaLogger.setParams({'Type':msgType}) # get logger tmpLogger = tmpPandaLogger.getHttpLogger(jedi_config.master.loggername) # escape special characters if escapeChar: message = message.replace('<','<') message = message.replace('>','>') # add message message = self.monToken + ' ' + message if msgLevel=='error': tmpLogger.error(message) elif msgLevel=='warning': tmpLogger.warning(message) elif msgLevel=='info': tmpLogger.info(message) else: tmpLogger.debug(message) # release HTTP handler tmpPandaLogger.release() except: pass
def sendMsg(self, message, msgType, msgLevel="info"): try: # get logger tmpPandaLogger = PandaLogger() # lock HTTP handler tmpPandaLogger.lock() tmpPandaLogger.setParams({"Type": msgType}) # get logger tmpLogger = tmpPandaLogger.getHttpLogger(jedi_config.master.loggername) # add message message = self.monToken + " " + message if msgLevel == "error": tmpLogger.error(message) elif msgLevel == "warning": tmpLogger.warning(message) elif msgLevel == "info": tmpLogger.info(message) else: tmpLogger.debug(message) # release HTTP handler tmpPandaLogger.release() except: pass
import pickle import stomp from pandaserver.config import panda_config from pandaserver.brokerage.SiteMapper import SiteMapper from pandaserver.dataservice.Finisher import Finisher from pandaserver.dataservice import DataServiceUtils from pandaserver.srvcore.CoreUtils import commands_get_status_output import logging logging.basicConfig(level=logging.DEBUG) # logger from pandacommon.pandalogger.PandaLogger import PandaLogger _logger = PandaLogger().getLogger('fileCallbackListener') # keep PID pidFile = '%s/file_callback_listener.pid' % panda_config.logdir # overall timeout value overallTimeout = 60 * 59 # expiration time expirationTime = datetime.datetime.utcnow() + datetime.timedelta( minutes=overallTimeout) # kill whole process def catch_sig(sig, frame): try:
import pandaserver.brokerage.broker from pandaserver.dataservice import DynDataDistributer from pandaserver.dataservice.MailUtils import MailUtils from pandaserver.dataservice.Notifier import Notifier from pandaserver.taskbuffer.JobSpec import JobSpec from pandaserver.taskbuffer import JobUtils from pandaserver.userinterface import Client from pandaserver.dataservice.DDM import rucioAPI from pandaserver.dataservice.DataServiceUtils import select_scope from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper # logger _logger = PandaLogger().getLogger('EventPicker') class EventPicker: # constructor def __init__(self, taskBuffer, siteMapper, evpFileName, ignoreError): self.taskBuffer = taskBuffer self.siteMapper = siteMapper self.ignoreError = ignoreError self.evpFileName = evpFileName self.token = datetime.datetime.utcnow().isoformat(' ') # logger self.logger = LogWrapper(_logger, self.token) self.pd2p = DynDataDistributer.DynDataDistributer([], self.taskBuffer, self.siteMapper,
from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.taskbuffer import EventServiceUtils from pandaserver.taskbuffer import retryModule from pandaserver.taskbuffer import JobUtils import pandaserver.taskbuffer.ErrorCode import pandaserver.dataservice.ErrorCode from pandaserver.dataservice import Closer try: long except NameError: long = int # logger _logger = PandaLogger().getLogger('Adder') panda_config.setupPlugin() class AdderGen(object): # constructor def __init__(self, taskBuffer, jobID, jobStatus, attemptNr, ignoreTmpError=True, siteMapper=None, pid=None, prelock_pid=None,
import sys import threading import traceback from pandaserver.configurator import aux from pandaserver.configurator.aux import * from datetime import datetime, timedelta from pandaserver.config import panda_config from pandacommon.pandalogger.PandaLogger import PandaLogger from pandaserver.configurator import db_interface as dbif from pandaserver.taskbuffer.TaskBuffer import taskBuffer _logger = PandaLogger().getLogger('configurator') _session = dbif.get_session() # Definitions of roles WRITE_LAN = 'write_lan' READ_LAN = 'read_lan' DEFAULT = 'default' class Configurator(threading.Thread): def __init__(self): threading.Thread.__init__(self) if hasattr(panda_config, 'AGIS_URL_SITES'): self.AGIS_URL_SITES = panda_config.AGIS_URL_SITES else: self.AGIS_URL_SITES = 'http://atlas-agis-api.cern.ch/request/site/query/?json&vo_name=atlas&state=ACTIVE' _logger.debug('Getting site dump...')
from pandacommon.pandalogger.PandaLogger import PandaLogger from pandajedi.jedicore.MsgWrapper import MsgWrapper from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jediddm.DDMInterface import DDMInterface from pandajedi.jediorder.JobBroker import JobBroker from pandajedi.jediorder.JobSplitter import JobSplitter from pandajedi.jediorder.JobGenerator import JobGeneratorThread from pandajedi.jedicore.ThreadUtils import ThreadPool from pandajedi.jediorder.TaskSetupper import TaskSetupper import sys logger = PandaLogger().getLogger('JobGenerator') tmpLog = MsgWrapper(logger) tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) datasetIDs = None if len(sys.argv) > 2: datasetIDs = [int(sys.argv[2])]
import sys from threading import Lock from pandaserver.config import panda_config # logger from pandacommon.pandalogger.PandaLogger import PandaLogger _logger = PandaLogger().getLogger('Initializer') # initialize cx_Oracle using dummy connection to avoid "Unable to acquire Oracle environment handle" class Initializer: def __init__(self): self.lock = Lock() self.first = True def init(self): _logger.debug("init new=%s" % self.first) # do nothing when nDBConnection is 0 if panda_config.nDBConnection == 0: return True # lock self.lock.acquire() if self.first: self.first = False try: _logger.debug("connect") # connect if panda_config.backend == 'oracle': import cx_Oracle conn = cx_Oracle.connect(dsn=panda_config.dbhost,
import hashlib try: long except NameError: long = int import pandaserver.userinterface.Client as Client from pandaserver.taskbuffer.JobSpec import JobSpec from pandaserver.taskbuffer.FileSpec import FileSpec from pandaserver.taskbuffer.TaskBuffer import taskBuffer from pandaserver.config import panda_config from pandaserver.test.testutils import sendCommand from pandacommon.pandalogger.PandaLogger import PandaLogger _logger = PandaLogger().getLogger('testJobFlowATLAS') class JobFlowATLAS(object): """ Class to test the different states of a job. It has functions to submit, get, finish and get the status a job. Some of the functions include assert statements for nostests that check the output is as expected. """ __datasetName = 'panda.destDB.%s' % uuid.uuid1() __destName = None __jobList = [] __XMLTEMPLATE_BASE = """<?xml version="1.0" encoding="UTF-8" standalone="no" ?> <!-- ATLAS file meta-data catalog -->
import datetime import pandaserver.jobdispatcher.Protocol as Protocol from pandaserver.taskbuffer import ErrorCode from pandaserver.userinterface import Client from pandaserver.config import panda_config from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper try: long except NameError: long = int # logger _logger = PandaLogger().getLogger('Utils') # check if server is alive def isAlive(req): return "alive=yes" # extract name from DN def cleanUserID(id): try: up = re.compile('/(DC|O|OU|C|L)=[^\/]+') username = up.sub('', id) up2 = re.compile('/CN=[0-9]+') username = up2.sub('', username) up3 = re.compile(' [0-9]+')
from pandacommon.pandautils import PandaUtils from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.srvcore.CoreUtils import commands_get_status_output from pandaserver.taskbuffer.TaskBufferInterface import TaskBufferInterface try: long except NameError: long = int # password from pandaserver.config import panda_config # logger _logger = PandaLogger().getLogger('add') tmpLog = LogWrapper(_logger, None) tmpLog.debug("===================== start =====================") # overall timeout value overallTimeout = 20 # grace period try: gracePeriod = int(sys.argv[1]) except Exception: gracePeriod = 3 # current minute
# logger from pandacommon.pandalogger.PandaLogger import PandaLogger from pandajedi.jedicore.MsgWrapper import MsgWrapper from pandajedi.jedicore.JediTaskBufferInterface import JediTaskBufferInterface from pandajedi.jedirefine import RefinerUtils from pandajedi.jediorder.TaskBroker import TaskBroker from pandajedi.jediddm.DDMInterface import DDMInterface import sys logger = PandaLogger().getLogger('TaskBroker') tmpLog = MsgWrapper(logger) tbIF = JediTaskBufferInterface() tbIF.setupInterface() siteMapper = tbIF.getSiteMapper() ddmIF = DDMInterface() ddmIF.setupInterface() jediTaskID = int(sys.argv[1]) s, taskSpec = tbIF.getTaskWithID_JEDI(jediTaskID, False) body = TaskBroker(None, tbIF, ddmIF, taskSpec.vo, taskSpec.prodSourceLabel) body.initializeMods(tbIF, ddmIF) taskParam = tbIF.getTaskParamsWithID_JEDI(jediTaskID) taskParamMap = RefinerUtils.decodeJSON(taskParam)
import time import datetime import multiprocessing import traceback from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandacommon.pandautils.thread_utils import GenericThread, WeightedLists from pandaserver.config import panda_config from pandaserver.brokerage.SiteMapper import SiteMapper from pandaserver.taskbuffer.TaskBuffer import TaskBuffer from pandaserver.taskbuffer.TaskBufferInterface import TaskBufferInterface from pandaserver.dataservice.AdderGen import AdderGen # logger _logger = PandaLogger().getLogger('add_main') # main def main(argv=tuple(), tbuf=None, **kwargs): try: long except NameError: long = int prelock_pid = GenericThread().get_pid() tmpLog = LogWrapper(_logger, "<pid={}>".format(prelock_pid)) tmpLog.debug("===================== start =====================")
from pandaserver.taskbuffer import EventServiceUtils import pandaserver.taskbuffer.ErrorCode from pandacommon.pandalogger.PandaLogger import PandaLogger from pandaserver.brokerage.SiteMapper import SiteMapper from pandaserver.dataservice.Finisher import Finisher from pandaserver.dataservice import DataServiceUtils from pandaserver.dataservice.DataServiceUtils import select_scope from pandaserver.dataservice.Closer import Closer import pandaserver.taskbuffer.ErrorCode from pandaserver.srvcore.CoreUtils import commands_get_status_output # password from pandaserver.config import panda_config # logger _logger = PandaLogger().getLogger('datasetManager') _logger.debug("===================== start =====================") # memory checker def _memoryCheck(str): try: proc_status = '/proc/%d/status' % os.getpid() procfile = open(proc_status) name = "" vmSize = "" vmRSS = "" # extract Name,VmSize,VmRSS for line in procfile: if line.startswith("Name:"): name = line.split()[-1]
from pandaserver.taskbuffer import OraDBProxy as DBProxy from pandaserver.config import panda_config from pandaserver.taskbuffer.JobSpec import JobSpec from pandaserver.taskbuffer.FileSpec import FileSpec from pandaserver.taskbuffer.DatasetSpec import DatasetSpec from pandacommon.pandalogger.PandaLogger import PandaLogger try: long except NameError: long = int # logger _logger = PandaLogger().getLogger('ConBridge') # exception for normal termination class HarmlessEx(Exception): pass # terminate child process by itself when master has gone class Terminator(threading.Thread): # constructor def __init__(self, consock): threading.Thread.__init__(self) self.consock = consock
""" proxy for database connection """ import re import warnings from pandaserver.config import panda_config from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.taskbuffer import OraDBProxy warnings.filterwarnings('ignore') # logger _logger = PandaLogger().getLogger('EiDBProxy') # proxy class EiDBProxy(OraDBProxy.DBProxy): # constructor def __init__(self, useOtherError=False): OraDBProxy.DBProxy.__init__(self, useOtherError) # connect to DB (just for INTR) def connect(self, dbhost=panda_config.ei_dbhost, dbpasswd=panda_config.ei_dbpasswd, dbuser=panda_config.ei_dbuser, dbname=panda_config.ei_dbname,
from pandaserver.taskbuffer.OraDBProxy import DBProxy import socket from pandaserver.config import panda_config import time try: from urlparse import parse_qs except ImportError: from urllib.parse import parse_qs from pandacommon.pandalogger.PandaLogger import PandaLogger _logger = PandaLogger().getLogger('testGetCriteriaGlobalShares') from testutils import sendCommand def retrieveJob(site): function = "getJob" node = {} node['siteName'] = site node['mem'] = 1000 node['node'] = socket.getfqdn() data = sendCommand(function, node, _logger) jobD = parse_qs(data) # jobD indicates it's a job in dictionary format, not a JobSpec object return jobD if __name__ == "__main__": proxyS = DBProxy() proxyS.connect(panda_config.dbhost,panda_config.dbpasswd,panda_config.dbuser,panda_config.dbname) #proxyS.getCriteriaForGlobalShares('BNL-OSG')
''' email utilities ''' import sys import smtplib from pandaserver.config import panda_config from pandacommon.pandalogger.PandaLogger import PandaLogger # logger _logger = PandaLogger().getLogger('MailUtils') # wrapper to patch smtplib.stderr to send debug info to logger class StderrLogger(object): def __init__(self, tmpLog): self.tmpLog = tmpLog def write(self, message): message = message.strip() if message != '': self.tmpLog.debug(message) # wrapper of SMTP to redirect messages class MySMTP(smtplib.SMTP): def set_log(self, tmp_log): self.tmpLog = tmp_log try: self.org_stderr = getattr(smtplib, 'stderr')
import re import sys import datetime import traceback from pandaserver.taskbuffer.TaskBuffer import taskBuffer from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.brokerage.SiteMapper import SiteMapper # password from pandaserver.config import panda_config passwd = panda_config.dbpasswd # logger _logger = PandaLogger().getLogger('prioryMassage') tmpLog = LogWrapper(_logger) tmpLog.debug("================= start ==================") # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate sitemapper siteMapper = SiteMapper(taskBuffer) # get usage breakdown usageBreakDownPerUser = {} usageBreakDownPerSite = {} workingGroupList = [] for table in ['ATLAS_PANDA.jobsActive4', 'ATLAS_PANDA.jobsArchived4']: varMap = {}
from pandaserver.test.testutils import sendCommand from pandacommon.pandalogger.PandaLogger import PandaLogger _logger = PandaLogger().getLogger('testUpdateWorkerPilotStatus') node = {} node['workerID'] = 9139456 node['harvesterID'] = 'CERN_central_k8s' node['status'] = 'started' node['site'] = 'CERN' function = "updateWorkerPilotStatus" data = sendCommand(function, node, _logger) _logger.debug(data)
def __init__(self): self._logger = PandaLogger().getLogger('PilotStreaming') return
setup dataset ''' import sys import uuid import traceback import threading from pandaserver.config import panda_config from pandacommon.pandalogger.PandaLogger import PandaLogger from pandacommon.pandalogger.LogWrapper import LogWrapper from pandaserver.taskbuffer import EventServiceUtils # logger _logger = PandaLogger().getLogger('Setupper') panda_config.setupPlugin() # main class class Setupper(threading.Thread): # constructor def __init__(self, taskBuffer, jobs, resubmit=False, pandaDDM=False, ddmAttempt=0, forkRun=False, onlyTA=False,
''' activate job ''' from pandacommon.pandalogger.PandaLogger import PandaLogger # logger _logger = PandaLogger().getLogger('Activator') class Activator: # constructor def __init__(self, taskBuffer, dataset, enforce=False): self.dataset = dataset self.taskBuffer = taskBuffer self.enforce = enforce # to keep backward compatibility def start(self): self.run() def join(self): pass # main def run(self): _logger.debug("start: %s" % self.dataset.name) if self.dataset.status in ['completed', 'deleting', 'deleted' ] and not self.enforce: _logger.debug(" skip: %s" % self.dataset.name)
class PilotStreaming: def __init__(self): self._logger = PandaLogger().getLogger('PilotStreaming') return def run(self): """ Gets and iterates over ups queues, deciding the job requirements and sending these to Harvester via the command interface :return: """ # timing time_start = time.time() self._logger.debug('Start.') # get unified pilot streaming (ups) queues ups_queues = taskBuffer.ups_get_queues() self._logger.debug('UPS queues: {0}'.format(ups_queues)) # get worker stats worker_stats = taskBuffer.ups_load_worker_stats_legacy() for ups_queue in ups_queues: # get the worker and job stats for the queue try: tmp_worker_stats = worker_stats[ups_queue] self._logger.debug('worker_stats for queue {0}: {1}'.format( ups_queue, tmp_worker_stats)) # tmp_job_stats = job_stats[ups_queue] except KeyError: # skip queue if no data available self._logger.debug( 'No worker stats for queue {0}'.format(ups_queue)) continue new_workers_per_harvester = taskBuffer.ups_new_worker_distribution_legacy( ups_queue, tmp_worker_stats) self._logger.info('queue: {0}, results: {1}'.format( ups_queue, new_workers_per_harvester)) # variables for the harvester command command = '{0}:{1}'.format('SET_N_WORKERS', ups_queue) status = 'new' ack_requested = False lock_interval = None com_interval = None for harvester_id in new_workers_per_harvester: params = new_workers_per_harvester[harvester_id] taskBuffer.commandToHarvester(harvester_id, command, ack_requested, status, lock_interval, com_interval, params) # timing time_stop = time.time() self._logger.debug( 'Done. Pilot streaming took: {0} s'.format(time_stop - time_start)) return
def run(self, taskBuffer, aSiteMapper, holdingAna): # import from pandaserver.dataservice.AdderGen import AdderGen # get logger _logger = PandaLogger().getLogger('add_process') # get file list timeNow = datetime.datetime.utcnow() timeInt = datetime.datetime.utcnow() dirName = panda_config.logdir fileList = os.listdir(dirName) fileList.sort() # remove duplicated files tmpList = [] uMap = {} for file in fileList: match = re.search('^(\d+)_([^_]+)_.{36}(_\d+)*$', file) if match is not None: fileName = '%s/%s' % (dirName, file) id = match.group(1) jobStatus = match.group(2) if id in uMap: try: os.remove(fileName) except Exception: pass else: if jobStatus != EventServiceUtils.esRegStatus: uMap[id] = fileName if long(id) in holdingAna: # give a priority to buildJobs tmpList.insert(0, file) else: tmpList.append(file) nFixed = 50 randTmp = tmpList[nFixed:] random.shuffle(randTmp) fileList = tmpList[:nFixed] + randTmp # add while len(fileList) != 0: # time limit to avoid too many copyArchive running at the same time if (datetime.datetime.utcnow() - timeNow) > datetime.timedelta(minutes=overallTimeout): tmpLog.debug("time over in Adder session") break # get fileList if (datetime.datetime.utcnow() - timeInt) > datetime.timedelta(minutes=15): timeInt = datetime.datetime.utcnow() # get file fileList = os.listdir(dirName) fileList.sort() # remove duplicated files tmpList = [] uMap = {} for file in fileList: match = re.search('^(\d+)_([^_]+)_.{36}(_\d+)*$', file) if match is not None: fileName = '%s/%s' % (dirName, file) id = match.group(1) jobStatus = match.group(2) if id in uMap: try: os.remove(fileName) except Exception: pass else: if jobStatus != EventServiceUtils.esRegStatus: uMap[id] = fileName if long(id) in holdingAna: # give a priority to buildJob tmpList.insert(0, file) else: tmpList.append(file) fileList = tmpList # check if if PandaUtils.isLogRotating(5, 5): tmpLog.debug("terminate since close to log-rotate time") break # choose a file file = fileList.pop(0) # check format match = re.search('^(\d+)_([^_]+)_.{36}(_\d+)*$', file) if match is not None: fileName = '%s/%s' % (dirName, file) if not os.path.exists(fileName): continue try: modTime = datetime.datetime( *(time.gmtime(os.path.getmtime(fileName))[:7])) thr = None if (timeNow - modTime) > datetime.timedelta(hours=24): # last chance tmpLog.debug("Last Add File {0} : {1}".format( os.getpid(), fileName)) thr = AdderGen(taskBuffer, match.group(1), match.group(2), fileName, ignoreTmpError=False, siteMapper=aSiteMapper) elif (timeInt - modTime) > datetime.timedelta(minutes=gracePeriod): # add tmpLog.debug("Add File {0} : {1}".format( os.getpid(), fileName)) thr = AdderGen(taskBuffer, match.group(1), match.group(2), fileName, ignoreTmpError=True, siteMapper=aSiteMapper) if thr is not None: thr.run() except Exception: type, value, traceBack = sys.exc_info() tmpLog.error("%s %s" % (type, value))
try: from Queue import Queue except ImportError: from queue import Queue from pandaserver.taskbuffer import OraDBProxy as DBProxy import os import time import random from threading import Lock from pandaserver.config import panda_config from pandaserver.taskbuffer.ConBridge import ConBridge from pandacommon.pandalogger.PandaLogger import PandaLogger # logger _logger = PandaLogger().getLogger('DBProxyPool') class DBProxyPool: def __init__(self, dbhost, dbpasswd, nConnection, useTimeout=False, dbProxyClass=None): # crate lock for callers self.lock = Lock() self.callers = [] # create Proxies _logger.debug("init") self.proxyList = Queue(nConnection)
import re import os import warnings from pandacommon.pandalogger.PandaLogger import PandaLogger from pandaserver.config import panda_config try: long except NameError: long = int warnings.filterwarnings('ignore') # logger _logger = PandaLogger().getLogger('WrappedCursor') # convert SQL and parameters in_printf format def convert_query_in_printf_format(sql, var_dict, sql_conv_map): if sql in sql_conv_map: sql = sql_conv_map[sql] else: old_sql = sql # % sql = re.sub(r'%', r'%%', sql) # current date except for being used for interval if re.search(r'CURRENT_DATE\s*[\+-]', sql, flags=re.IGNORECASE) is None: sql = re.sub(r'CURRENT_DATE', r'CURRENT_TIMESTAMP',
""" provide web service for DDM """ import sys from pandaserver.taskbuffer.WrappedPickle import WrappedPickle from pandacommon.pandalogger.PandaLogger import PandaLogger # logger _logger = PandaLogger().getLogger('DataService') class DataService: # constructor def __init__(self): self.taskBuffer = None # set taskbuffer def init(self, taskBuffer): self.taskBuffer = taskBuffer # Singleton dataService = DataService() del DataService ''' web interface '''
import sys import time import datetime from pandajedi.jedicore.ThreadUtils import ListWithLock, ThreadPool, WorkerThread from pandajedi.jedicore import Interaction from pandajedi.jedicore.MsgWrapper import MsgWrapper from pandajedi.jedicore.FactoryBase import FactoryBase from .JediKnight import JediKnight from pandajedi.jediconfig import jedi_config # logger from pandacommon.pandalogger.PandaLogger import PandaLogger logger = PandaLogger().getLogger(__name__.split('.')[-1]) # worker class to refine TASK_PARAM to fill JEDI tables class TaskBroker(JediKnight, FactoryBase): # constructor def __init__(self, commuChannel, taskBufferIF, ddmIF, vos, prodSourceLabels): self.vos = self.parseInit(vos) self.prodSourceLabels = self.parseInit(prodSourceLabels) JediKnight.__init__(self, commuChannel, taskBufferIF, ddmIF, logger) FactoryBase.__init__(self, self.vos, self.prodSourceLabels, logger, jedi_config.taskbroker.modConfig) # main
import re import sys import glob import time import os.path import datetime import threading from pandaserver.config import panda_config from pandaserver.taskbuffer.TaskBuffer import taskBuffer from pandaserver.brokerage import SiteMapper from pandaserver.dataservice.EventPicker import EventPicker from pandacommon.pandalogger.PandaLogger import PandaLogger from pandaserver.srvcore.CoreUtils import commands_get_status_output # logger _logger = PandaLogger().getLogger('evpPD2P') _logger.debug("===================== start =====================") # overall timeout value overallTimeout = 300 # prefix of evp files prefixEVP = 'evp.' # file pattern of evp files evpFilePatt = panda_config.cache_dir + '/' + prefixEVP + '*' # kill old process try: # time limit timeLimit = datetime.datetime.utcnow() - datetime.timedelta(minutes=overallTimeout) # get process list