def __init__(self,
taskBuffer,
jobID,
jobStatus,
xmlFile,
ignoreTmpError=True,
siteMapper=None):
self.job = None
self.jobID = jobID
self.jobStatus = jobStatus
self.taskBuffer = taskBuffer
self.ignoreTmpError = ignoreTmpError
self.lockXML = None
self.siteMapper = siteMapper
self.attemptNr = None
self.xmlFile = xmlFile
self.datasetMap = {}
self.extraInfo = {
'surl': {},
'nevents': {},
'lbnr': {},
'endpoint': {}
}
# exstract attemptNr
try:
tmpAttemptNr = self.xmlFile.split('/')[-1].split('_')[-1]
if re.search('^\d+$', tmpAttemptNr) != None:
self.attemptNr = int(tmpAttemptNr)
except:
pass
# logger
self.logger = LogWrapper(_logger, str(self.jobID))
def run(self):
# get logger
tmpLog = LogWrapper(_logger,'<vuid={0} site={1} name={2}>'.format(self.vuid,
self.site,
self.dataset))
# query dataset
tmpLog.debug("start")
if self.vuid != None:
dataset = self.taskBuffer.queryDatasetWithMap({'vuid':self.vuid})
else:
dataset = self.taskBuffer.queryDatasetWithMap({'name':self.dataset})
if dataset == None:
tmpLog.error("Not found")
tmpLog.debug("end")
return
tmpLog.debug("type:%s name:%s" % (dataset.type,dataset.name))
if dataset.type == 'dispatch':
# activate jobs in jobsDefined
Activator(self.taskBuffer,dataset).start()
if dataset.type == 'output':
if dataset.name != None and re.search('^panda\..*_zip$',dataset.name) != None:
# start unmerge jobs
Activator(self.taskBuffer,dataset,enforce=True).start()
else:
# finish transferring jobs
Finisher(self.taskBuffer,dataset,site=self.site).start()
tmpLog.debug("end")
def make_logger(tmp_log, token=None, method_name=None, hook=None):
# get method name of caller
if method_name is None:
tmpStr = inspect.stack()[1][3]
else:
tmpStr = method_name
if token is not None:
tmpStr += ' <{0}>'.format(token)
else:
tmpStr += ' :'.format(token)
newLog = LogWrapper(tmp_log, tmpStr, seeMem=with_memory_profile, hook=hook)
return newLog
def uploadLog(req, file):
if not Protocol.isSecure(req):
return False
if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']:
return False
tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename))
tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN']))
# size check
sizeLimit = 100 * 1024 * 1024
# get file size
contentLength = 0
try:
contentLength = long(req.headers_in["content-length"])
except:
if req.headers_in.has_key("content-length"):
tmpLog.error("cannot get CL : %s" %
req.headers_in["content-length"])
else:
tmpLog.error("no CL")
tmpLog.debug("size %s" % contentLength)
if contentLength > sizeLimit:
errStr = "failed to upload log due to size limit"
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
jediLogDir = '/jedilog'
retStr = ''
try:
fileBaseName = file.filename.split('/')[-1]
fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir,
fileBaseName)
# delete old file
if os.path.exists(fileFullPath):
os.remove(fileFullPath)
# write
fo = open(fileFullPath, 'wb')
fileContent = file.file.read()
fo.write(fileContent)
fo.close()
tmpLog.debug("written to {0}".format(fileFullPath))
retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None),
jediLogDir, fileBaseName)
except:
errtype, errvalue = sys.exc_info()[:2]
errStr = "failed to write log with {0}:{1}".format(
errtype.__name__, errvalue)
tmpLog.error(errStr)
tmpLog.debug("end")
return errStr
tmpLog.debug("end")
return retStr
def __init__(self,taskBuffer,siteMapper,evpFileName,ignoreError):
self.taskBuffer = taskBuffer
self.siteMapper = siteMapper
self.ignoreError = ignoreError
self.evpFileName = evpFileName
self.token = datetime.datetime.utcnow().isoformat(' ')
# logger
self.logger = LogWrapper(_logger,self.token)
self.pd2p = DynDataDistributer.DynDataDistributer([],self.taskBuffer,self.siteMapper,
token=' ',logger=self.logger)
self.userDatasetName = ''
self.creationTime = ''
self.params = ''
self.lockedBy = ''
self.evpFile = None
self.userTaskName = ''
# message buffer
self.msgBuffer = []
self.lineLimit = 100
# JEDI
self.jediTaskID = None
import multiprocessing
from taskbuffer.TaskBuffer import taskBuffer
import pandalogger.PandaLogger
from pandalogger.PandaLogger import PandaLogger
from brokerage.SiteMapper import SiteMapper
from pandautils import PandaUtils
from pandalogger.LogWrapper import LogWrapper
# password
from config import panda_config
passwd = panda_config.dbpasswd
# logger
_logger = PandaLogger().getLogger('add')
tmpLog = LogWrapper(_logger, None)
tmpLog.debug("===================== start =====================")
# overall timeout value
overallTimeout = 20
# grace period
try:
gracePeriod = int(sys.argv[1])
except:
gracePeriod = 3
# current minute
currentMinute = datetime.datetime.utcnow().minute
import re
from config import panda_config
from taskbuffer.TaskBuffer import taskBuffer
from pandalogger.PandaLogger import PandaLogger
from pandalogger.LogWrapper import LogWrapper
import panda_proxy_cache
# logger
_logger = PandaLogger().getLogger('panda_activeusers_query')
tmpLog = LogWrapper(_logger)
if __name__ == '__main__':
tmpLog.debug("================= start ==================")
# instantiate TB
taskBuffer.init(panda_config.dbhost,
panda_config.dbpasswd,
nDBConnection=1)
# instantiate MyProxy I/F
my_proxy_interface_instance = panda_proxy_cache.MyProxyInterface()
# roles
if hasattr(panda_config, 'proxy_cache_roles'):
roles = panda_config.proxy_cache_roles.split(',')
else:
roles = [
'atlas', 'atlas:/atlas/Role=production', 'atlas:/atlas/Role=pilot'
def updateJob(req,
jobId,
state,
token=None,
transExitCode=None,
pilotErrorCode=None,
pilotErrorDiag=None,
timestamp=None,
timeout=60,
xml='',
node=None,
workdir=None,
cpuConsumptionTime=None,
cpuConsumptionUnit=None,
remainingSpace=None,
schedulerID=None,
pilotID=None,
siteName=None,
messageLevel=None,
pilotLog='',
metaData='',
cpuConversionFactor=None,
exeErrorCode=None,
exeErrorDiag=None,
pilotTiming=None,
computingElement=None,
startTime=None,
endTime=None,
nEvents=None,
nInputFiles=None,
batchID=None,
attemptNr=None,
jobMetrics=None,
stdout='',
jobSubStatus=None,
coreCount=None,
maxRSS=None,
maxVMEM=None,
maxSWAP=None,
maxPSS=None,
avgRSS=None,
avgVMEM=None,
avgSWAP=None,
avgPSS=None):
tmpLog = LogWrapper(
_logger, 'updateJob PandaID={0} PID={1}'.format(jobId, os.getpid()))
tmpLog.debug('start')
# get DN
realDN = _getDN(req)
# get FQANs
fqans = _getFQAN(req)
# check production role
prodManager = _checkRole(fqans,
realDN,
jobDispatcher,
site=siteName,
hostname=req.get_remote_host())
# check token
validToken = _checkToken(token, jobDispatcher)
# accept json
acceptJson = req.acceptJson()
_logger.debug(
"updateJob(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,attemptNr:%s,jobSubStatus:%s,core:%s,DN:%s,role:%s,token:%s,val:%s,FQAN:%s,maxRSS=%s,maxVMEM=%s,maxSWAP=%s,maxPSS=%s,avgRSS=%s,avgVMEM=%s,avgSWAP=%s,avgPSS=%s\n==XML==\n%s\n==LOG==\n%s\n==Meta==\n%s\n==Metrics==\n%s\n==stdout==\n%s)"
% (jobId, state, transExitCode, pilotErrorCode, pilotErrorDiag, node,
workdir, cpuConsumptionTime, cpuConsumptionUnit, remainingSpace,
schedulerID, pilotID, siteName, messageLevel, nEvents, nInputFiles,
cpuConversionFactor, exeErrorCode, exeErrorDiag, pilotTiming,
computingElement, startTime, endTime, batchID, attemptNr,
jobSubStatus, coreCount, realDN, prodManager, token, validToken,
str(fqans), maxRSS, maxVMEM, maxSWAP, maxPSS, avgRSS, avgVMEM,
avgSWAP, avgPSS, xml, pilotLog, metaData, jobMetrics, stdout))
_pilotReqLogger.info('method=updateJob,site=%s,node=%s,type=None' %
(siteName, node))
# invalid role
if not prodManager:
_logger.warning("updateJob(%s) : invalid role" % jobId)
return Protocol.Response(Protocol.SC_Role).encode(acceptJson)
# invalid token
if not validToken:
_logger.warning("updateJob(%s) : invalid token" % jobId)
return Protocol.Response(Protocol.SC_Invalid).encode(acceptJson)
# aborting message
if jobId == 'NULL':
return Protocol.Response(Protocol.SC_Success).encode(acceptJson)
# check status
if not state in [
'running', 'failed', 'finished', 'holding', 'starting',
'transferring'
]:
_logger.warning("invalid state=%s for updateJob" % state)
return Protocol.Response(Protocol.SC_Success).encode(acceptJson)
# pilot log
tmpLog.debug('sending log')
if pilotLog != '':
try:
# make message
message = pilotLog
# get logger
_pandaLogger = PandaLogger()
_pandaLogger.lock()
_pandaLogger.setParams({'Type': 'pilotLog', 'PandaID': int(jobId)})
logger = _pandaLogger.getHttpLogger(panda_config.loggername)
# add message
logger.info(message)
except:
tmpLog.debug('failed to send log')
finally:
tmpLog.debug('release lock')
try:
# release HTTP handler
_pandaLogger.release()
except:
pass
tmpLog.debug('done log')
# create parameter map
param = {}
if cpuConsumptionTime != None:
param['cpuConsumptionTime'] = cpuConsumptionTime
if cpuConsumptionUnit != None:
param['cpuConsumptionUnit'] = cpuConsumptionUnit
if node != None:
param['modificationHost'] = node[:128]
if transExitCode != None:
param['transExitCode'] = transExitCode
if pilotErrorCode != None:
param['pilotErrorCode'] = pilotErrorCode
if pilotErrorDiag != None:
param['pilotErrorDiag'] = pilotErrorDiag[:500]
if jobMetrics != None:
param['jobMetrics'] = jobMetrics[:500]
if schedulerID != None:
param['schedulerID'] = schedulerID
if pilotID != None:
param['pilotID'] = pilotID[:200]
if batchID != None:
param['batchID'] = batchID[:80]
if exeErrorCode != None:
param['exeErrorCode'] = exeErrorCode
if exeErrorDiag != None:
param['exeErrorDiag'] = exeErrorDiag[:500]
if cpuConversionFactor != None:
param['cpuConversion'] = cpuConversionFactor
if pilotTiming != None:
param['pilotTiming'] = pilotTiming
if computingElement != None:
param['computingElement'] = computingElement
if nEvents != None:
param['nEvents'] = nEvents
if nInputFiles != None:
param['nInputFiles'] = nInputFiles
if not jobSubStatus in [None, '']:
param['jobSubStatus'] = jobSubStatus
if not coreCount in [None, '']:
param['actualCoreCount'] = coreCount
if maxRSS != None:
param['maxRSS'] = maxRSS
if maxVMEM != None:
param['maxVMEM'] = maxVMEM
if maxSWAP != None:
param['maxSWAP'] = maxSWAP
if maxPSS != None:
param['maxPSS'] = maxPSS
if avgRSS != None:
param['avgRSS'] = avgRSS
if avgVMEM != None:
param['avgVMEM'] = avgVMEM
if avgSWAP != None:
param['avgSWAP'] = avgSWAP
if avgPSS != None:
param['avgPSS'] = avgPSS
if startTime != None:
try:
param['startTime'] = datetime.datetime(
*time.strptime(startTime, '%Y-%m-%d %H:%M:%S')[:6])
except:
pass
if endTime != None:
try:
param['endTime'] = datetime.datetime(
*time.strptime(endTime, '%Y-%m-%d %H:%M:%S')[:6])
except:
pass
if attemptNr != None:
try:
attemptNr = int(attemptNr)
except:
attemptNr = None
if stdout != '':
stdout = stdout[:2048]
# invoke JD
tmpLog.debug('executing')
return jobDispatcher.updateJob(int(jobId), state, int(timeout), xml,
siteName, param, metaData, attemptNr,
stdout, acceptJson)
def application(environ, start_response):
# get method name
methodName = ''
if environ.has_key('SCRIPT_NAME'):
methodName = environ['SCRIPT_NAME'].split('/')[-1]
tmpLog = LogWrapper(_logger,
"PID={0} {1}".format(os.getpid(), methodName))
tmpLog.debug("start")
regStart = datetime.datetime.utcnow()
retType = None
# check method name
if not methodName in allowedMethods:
tmpLog.error("is forbidden")
exeRes = "False : %s is forbidden" % methodName
else:
# get method object
tmpMethod = None
try:
exec "tmpMethod = %s" % methodName
except:
pass
# object not found
if tmpMethod == None:
tmpLog.error("is undefined")
exeRes = "False"
else:
try:
# get params
tmpPars = cgi.FieldStorage(environ['wsgi.input'],
environ=environ,
keep_blank_values=1)
# convert to map
params = {}
for tmpKey in tmpPars.keys():
if tmpPars[tmpKey].file != None and tmpPars[
tmpKey].filename != None:
# file
params[tmpKey] = tmpPars[tmpKey]
else:
# string
params[tmpKey] = tmpPars.getfirst(tmpKey)
if panda_config.entryVerbose:
tmpLog.debug("with %s" % str(params.keys()))
# dummy request object
dummyReq = DummyReq(environ, tmpLog)
# exec
exeRes = apply(tmpMethod, [dummyReq], params)
# extract return type
if type(exeRes) == types.DictType:
retType = exeRes['type']
exeRes = exeRes['content']
# convert bool to string
if exeRes in [True, False]:
exeRes = str(exeRes)
except Exception as e:
tmpLog.error("execution failure : {0}".format(str(e)))
errStr = ""
for tmpKey, tmpVal in environ.iteritems():
errStr += "%s : %s\n" % (tmpKey, str(tmpVal))
tmpLog.error(errStr)
# return internal server error
start_response('500 INTERNAL SERVER ERROR',
[('Content-Type', 'text/plain')])
return [str(e)]
if panda_config.entryVerbose:
tmpLog.debug("done")
regTime = datetime.datetime.utcnow() - regStart
tmpLog.info(
"exec_time=%s.%03d sec, return len=%s B" %
(regTime.seconds, regTime.microseconds / 1000, len(str(exeRes))))
# return
if exeRes == taskbuffer.ErrorCode.EC_NotFound:
start_response('404 Not Found', [('Content-Type', 'text/plain')])
return ['not found']
elif isinstance(exeRes, taskbuffer.ErrorCode.EC_Redirect):
start_response('302 Redirect', [('Location', exeRes.url)])
return ['redirect']
else:
if retType == 'json':
start_response('200 OK',
[('Content-Type', 'application/json')])
else:
start_response('200 OK', [('Content-Type', 'text/plain')])
return [exeRes]
def __init__(self,job,datasets,log):
self.jobSpec = job
self.datasets = datasets
self.tmpLog = LogWrapper(log,"{0} CloserAtlasPlugin".format(self.jobSpec.PandaID))
def updateJob(req,
jobId,
state,
token=None,
transExitCode=None,
pilotErrorCode=None,
pilotErrorDiag=None,
timestamp=None,
timeout=60,
xml='',
node=None,
workdir=None,
cpuConsumptionTime=None,
cpuConsumptionUnit=None,
remainingSpace=None,
schedulerID=None,
pilotID=None,
siteName=None,
messageLevel=None,
pilotLog='',
metaData='',
cpuConversionFactor=None,
exeErrorCode=None,
exeErrorDiag=None,
pilotTiming=None,
computingElement=None,
startTime=None,
endTime=None,
nEvents=None,
nInputFiles=None,
batchID=None,
attemptNr=None,
jobMetrics=None,
stdout='',
jobSubStatus=None,
coreCount=None,
maxRSS=None,
maxVMEM=None,
maxSWAP=None,
maxPSS=None,
avgRSS=None,
avgVMEM=None,
avgSWAP=None,
avgPSS=None,
totRCHAR=None,
totWCHAR=None,
totRBYTES=None,
totWBYTES=None,
rateRCHAR=None,
rateWCHAR=None,
rateRBYTES=None,
rateWBYTES=None):
tmpLog = LogWrapper(
_logger, 'updateJob PandaID={0} PID={1}'.format(jobId, os.getpid()))
tmpLog.debug('start')
# get DN
realDN = _getDN(req)
# get FQANs
fqans = _getFQAN(req)
# check production role
prodManager = _checkRole(fqans,
realDN,
jobDispatcher,
site=siteName,
hostname=req.get_remote_host())
# check token
validToken = _checkToken(token, jobDispatcher)
# accept json
acceptJson = req.acceptJson()
_logger.debug(
"updateJob(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,attemptNr:%s,jobSubStatus:%s,core:%s,DN:%s,role:%s,token:%s,val:%s,FQAN:%s,maxRSS=%s,maxVMEM=%s,maxSWAP=%s,maxPSS=%s,avgRSS=%s,avgVMEM=%s,avgSWAP=%s,avgPSS=%s,totRCHAR=%s,totWCHAR=%s,totRBYTES=%s,totWBYTES=%s,rateRCHAR=%s,rateWCHAR=%s,rateRBYTES=%s,rateWBYTES=%s\n==XML==\n%s\n==LOG==\n%s\n==Meta==\n%s\n==Metrics==\n%s\n==stdout==\n%s)"
% (jobId, state, transExitCode, pilotErrorCode, pilotErrorDiag, node,
workdir, cpuConsumptionTime, cpuConsumptionUnit, remainingSpace,
schedulerID, pilotID, siteName, messageLevel, nEvents, nInputFiles,
cpuConversionFactor, exeErrorCode, exeErrorDiag, pilotTiming,
computingElement, startTime, endTime, batchID, attemptNr,
jobSubStatus, coreCount, realDN, prodManager, token, validToken,
str(fqans), maxRSS, maxVMEM, maxSWAP, maxPSS, avgRSS, avgVMEM,
avgSWAP, avgPSS, totRCHAR, totWCHAR, totRBYTES, totWBYTES,
rateRCHAR, rateWCHAR, rateRBYTES, rateWBYTES, xml, pilotLog[:1024],
metaData[:1024], jobMetrics, stdout))
_pilotReqLogger.debug('method=updateJob,site=%s,node=%s,type=None' %
(siteName, node))
# invalid role
if not prodManager:
_logger.warning("updateJob(%s) : invalid role" % jobId)
if acceptJson:
tmpMsg = 'no production/pilot role in VOMS FQANs or non pilot owner'
else:
tmpMsg = None
return Protocol.Response(Protocol.SC_Role, tmpMsg).encode(acceptJson)
# invalid token
if not validToken:
_logger.warning("updateJob(%s) : invalid token" % jobId)
return Protocol.Response(Protocol.SC_Invalid).encode(acceptJson)
# aborting message
if jobId == 'NULL':
return Protocol.Response(Protocol.SC_Success).encode(acceptJson)
# check status
if not state in [
'running', 'failed', 'finished', 'holding', 'starting',
'transferring'
]:
_logger.warning("invalid state=%s for updateJob" % state)
return Protocol.Response(Protocol.SC_Success).encode(acceptJson)
# create parameter map
param = {}
if cpuConsumptionTime != None:
param['cpuConsumptionTime'] = cpuConsumptionTime
if cpuConsumptionUnit != None:
param['cpuConsumptionUnit'] = cpuConsumptionUnit
if node != None:
param['modificationHost'] = node[:128]
if transExitCode != None:
param['transExitCode'] = transExitCode
if pilotErrorCode != None:
param['pilotErrorCode'] = pilotErrorCode
if pilotErrorDiag != None:
param['pilotErrorDiag'] = pilotErrorDiag[:500]
if jobMetrics != None:
param['jobMetrics'] = jobMetrics[:500]
if schedulerID != None:
param['schedulerID'] = schedulerID
if pilotID != None:
param['pilotID'] = pilotID[:200]
if batchID != None:
param['batchID'] = batchID[:80]
if exeErrorCode != None:
param['exeErrorCode'] = exeErrorCode
if exeErrorDiag != None:
param['exeErrorDiag'] = exeErrorDiag[:500]
if cpuConversionFactor != None:
param['cpuConversion'] = cpuConversionFactor
if pilotTiming != None:
param['pilotTiming'] = pilotTiming
if computingElement != None:
param['computingElement'] = computingElement
if nEvents != None:
param['nEvents'] = nEvents
if nInputFiles != None:
param['nInputFiles'] = nInputFiles
if not jobSubStatus in [None, '']:
param['jobSubStatus'] = jobSubStatus
if not coreCount in [None, '']:
param['actualCoreCount'] = coreCount
if maxRSS != None:
param['maxRSS'] = maxRSS
if maxVMEM != None:
param['maxVMEM'] = maxVMEM
if maxSWAP != None:
param['maxSWAP'] = maxSWAP
if maxPSS != None:
param['maxPSS'] = maxPSS
if avgRSS != None:
param['avgRSS'] = avgRSS
if avgVMEM != None:
param['avgVMEM'] = avgVMEM
if avgSWAP != None:
param['avgSWAP'] = avgSWAP
if avgPSS != None:
param['avgPSS'] = avgPSS
if totRCHAR is not None:
totRCHAR = int(totRCHAR) / 1024 # convert to kByte
totRCHAR = min(10**10 - 1, totRCHAR) # limit to 10 digit
param['totRCHAR'] = totRCHAR
if totWCHAR is not None:
totWCHAR = int(totWCHAR) / 1024 # convert to kByte
totWCHAR = min(10**10 - 1, totWCHAR) # limit to 10 digit
param['totWCHAR'] = totWCHAR
if totRBYTES is not None:
totRBYTES = int(totRBYTES) / 1024 # convert to kByte
totRBYTES = min(10**10 - 1, totRBYTES) # limit to 10 digit
param['totRBYTES'] = totRBYTES
if totWBYTES is not None:
totWBYTES = int(totWBYTES) / 1024 # convert to kByte
totWBYTES = min(10**10 - 1, totWBYTES) # limit to 10 digit
param['totWBYTES'] = totWBYTES
if rateRCHAR is not None:
rateRCHAR = min(10**10 - 1, int(rateRCHAR)) # limit to 10 digit
param['rateRCHAR'] = rateRCHAR
if rateWCHAR is not None:
rateWCHAR = min(10**10 - 1, int(rateWCHAR)) # limit to 10 digit
param['rateWCHAR'] = rateWCHAR
if rateRBYTES is not None:
rateRBYTES = min(10**10 - 1, int(rateRBYTES)) # limit to 10 digit
param['rateRBYTES'] = rateRBYTES
if rateWBYTES is not None:
rateWBYTES = min(10**10 - 1, int(rateWBYTES)) # limit to 10 digit
param['rateWBYTES'] = rateWBYTES
if startTime != None:
try:
param['startTime'] = datetime.datetime(
*time.strptime(startTime, '%Y-%m-%d %H:%M:%S')[:6])
except:
pass
if endTime != None:
try:
param['endTime'] = datetime.datetime(
*time.strptime(endTime, '%Y-%m-%d %H:%M:%S')[:6])
except:
pass
if attemptNr != None:
try:
attemptNr = int(attemptNr)
except:
attemptNr = None
if stdout != '':
stdout = stdout[:2048]
# invoke JD
tmpLog.debug('executing')
return jobDispatcher.updateJob(int(jobId), state, int(timeout), xml,
siteName, param, metaData, pilotLog,
attemptNr, stdout, acceptJson)
def run(self):
try:
# make a message instance
tmpLog = LogWrapper(_logger, None)
# run main procedure in the same process
if not self.forkRun:
tmpLog.debug('main start')
tmpLog.debug('firstSubmission={0}'.format(
self.firstSubmission))
# group jobs per VO
voJobsMap = {}
ddmFreeJobs = []
tmpLog.debug('{0} jobs in total'.format(len(self.jobs)))
for tmpJob in self.jobs:
# set VO=local for DDM free
if tmpJob.destinationSE == 'local':
tmpVO = 'local'
else:
tmpVO = tmpJob.VO
# make map
if not voJobsMap.has_key(tmpVO):
voJobsMap[tmpVO] = []
voJobsMap[tmpVO].append(tmpJob)
# loop over all VOs
for tmpVO, tmpJobList in voJobsMap.iteritems():
tmpLog.debug('vo={0} has {1} jobs'.format(
tmpVO, len(tmpJobList)))
# get plugin
setupperPluginClass = panda_config.getPlugin(
'setupper_plugins', tmpVO)
if setupperPluginClass == None:
# use ATLAS plug-in by default
from SetupperAtlasPlugin import SetupperAtlasPlugin
setupperPluginClass = SetupperAtlasPlugin
tmpLog.debug('plugin name -> {0}'.format(
setupperPluginClass.__name__))
try:
# make plugin
setupperPlugin = setupperPluginClass(
self.taskBuffer,
self.jobs,
tmpLog,
resubmit=self.resubmit,
pandaDDM=self.pandaDDM,
ddmAttempt=self.ddmAttempt,
onlyTA=self.onlyTA,
firstSubmission=self.firstSubmission)
# run plugin
tmpLog.debug('run plugin')
setupperPlugin.run()
# go forward if not TA
if not self.onlyTA:
# update jobs
tmpLog.debug('update jobs')
self.updateJobs(
setupperPlugin.jobs + setupperPlugin.jumboJobs,
tmpLog)
# execute post process
tmpLog.debug('post execute plugin')
setupperPlugin.postRun()
tmpLog.debug('done plugin')
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('plugin failed with {0}:{1}'.format(
errtype, errvalue))
tmpLog.debug('main end')
else:
tmpLog.debug('fork start')
# write jobs to file
import os
import cPickle as pickle
outFileName = '%s/set.%s_%s' % (panda_config.logdir,
self.jobs[0].PandaID,
commands.getoutput('uuidgen'))
outFile = open(outFileName, 'w')
pickle.dump(self.jobs, outFile)
outFile.close()
# run main procedure in another process because python doesn't release memory
com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % (
panda_config.home_dir_cwd, panda_config.home_dir_cwd)
com += 'source %s; ' % panda_config.glite_source
com += 'env PYTHONPATH=%s:%s %s/python -Wignore %s/dataservice/forkSetupper.py -i %s' % \
(panda_config.pandaCommon_dir,panda_config.pandaPython_dir,panda_config.native_python,
panda_config.pandaPython_dir,outFileName)
if self.onlyTA:
com += " -t"
if not self.firstSubmission:
com += " -f"
tmpLog.debug(com)
# exeute
status, output = self.taskBuffer.processLimiter.getstatusoutput(
com)
tmpLog.debug("return from main process: %s %s" %
(status, output))
tmpLog.debug('fork end')
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('master failed with {0}:{1}'.format(
errtype, errvalue))
def getGUIDsFromEventIndex(self, runEventList, streamName, amiTags,
dataType):
comment = ' /* DBProxy.getGUIDsFromEventIndex */'
methodName = comment.split(' ')[-2].split('.')[-1]
tmpLog = LogWrapper(
_logger,
methodName + " <streamName={0} amiTags={1} dataType={2}>".format(
streamName, amiTags, dataType))
try:
# change to list
if not amiTags in [None, '']:
amiTags = amiTags.replace('*', '.*').split(',')
tmpLog.debug("start for {0} events".format(len(runEventList)))
# check data type
if not dataType in ['RAW', 'ESD', 'AOD']:
return False, 'dataType={0} is unsupported'.format(dataType)
# sql to insert runs and events
sqlRE = "INSERT INTO {0}.TMP_RUN_EVENT_PAIRS (runNumber,eventNumber) ".format(
panda_config.schemaEI)
sqlRE += "VALUES (:runNumber,:eventNumber) "
varMaps = []
for runNumber, eventNumber in runEventList:
varMap = {}
varMap[':runNumber'] = runNumber
varMap[':eventNumber'] = eventNumber
varMaps.append(varMap)
# begin transaction
self.conn.begin()
self.cur.arraysize = 100000
# insert runs and events
self.cur.executemany(sqlRE + comment, varMaps)
# read GUIDs
varMap = {}
if amiTags in [None, '']:
sqlRG = "SELECT runNumber,eventNumber,guid_{0} ".format(
dataType)
sqlRG += "FROM {0}.V_PANDA_EVPICK_NOAMITAG_MANY ".format(
panda_config.schemaEI)
else:
sqlRG = "SELECT runNumber,eventNumber,guid_{0},amiTag ".format(
dataType)
sqlRG += "FROM {0}.V_PANDA_EVPICK_AMITAG_MANY ".format(
panda_config.schemaEI)
if not streamName in [None, '']:
sqlRG += "WHERE streamName=:streamName "
varMap[':streamName'] = streamName
self.cur.execute(sqlRG + comment, varMap)
resRG = self.cur.fetchall()
# commit
if not self._commit():
raise RuntimeError, 'Commit error'
retValue = {}
keyAmiIdxMap = {}
for tmpItem in resRG:
if amiTags in [None, '']:
runNumber, eventNumber, guid = tmpItem
# dummy
idxTag = 0
else:
runNumber, eventNumber, guid, amiTag = tmpItem
# get index number for the AMI tag in the list
idxTag = self.getIndexAmiTag(amiTags, amiTag)
# didn't match
if idxTag == None:
continue
tmpKey = (runNumber, eventNumber)
# use AMI tag in a preference orde
if tmpKey in keyAmiIdxMap and keyAmiIdxMap[tmpKey] < idxTag:
continue
keyAmiIdxMap[tmpKey] = idxTag
retValue[tmpKey] = [guid]
tmpLog.debug("found {0} events".format(len(retValue)))
return True, retValue
except:
# roll back
self._rollback()
# error
self.dumpErrorMessage(_logger, methodName)
return False, None
def getFilesFromLRC(files,
url,
guids=[],
storageName=[],
terminateWhenFailed=False,
getPFN=False,
scopeList=[]):
tmpLog = LogWrapper(_log, None)
tmpLog.debug('getFilesFromLRC "%s" %s' % (url, str(storageName)))
# get PFC
outSTR = ''
if url.startswith('mysql://'):
# from MySQL
outSTR = _getPFNFromMySQL(files, url)
# get PFN
if getPFN:
outPFN = {}
# FIXME
tmpLog.debug('RetPFN:%s ' % str(outPFN))
return outPFN
elif url.startswith('http://'):
# from HTTP I/F
outSTR = _getPoolFileCatalog(files, url)
# get PFN
if getPFN:
outPFN = {}
try:
if not outSTR in ['', None]:
root = xml.dom.minidom.parseString(outSTR)
fileNodes = root.getElementsByTagName('File')
for file in fileNodes:
# get PFN and LFN nodes
physical = file.getElementsByTagName('physical')[0]
pfnNode = physical.getElementsByTagName('pfn')[0]
logical = file.getElementsByTagName('logical')[0]
lfnNode = logical.getElementsByTagName('lfn')[0]
# convert UTF8 to Raw
pfn = str(pfnNode.getAttribute('name'))
lfn = str(lfnNode.getAttribute('name'))
# assign
if not outPFN.has_key(lfn):
outPFN[lfn] = []
outPFN[lfn].append(pfn)
except:
type, value, traceBack = sys.exc_info()
tmpLog.error(outSTR)
tmpLog.error("could not parse XML - %s %s" % (type, value))
tmpLog.debug('RetPFN:%s ' % str(outPFN))
return outPFN
elif url.startswith('lfc://') or url.startswith('rucio://'):
# from LFC
timeStart = datetime.datetime.utcnow()
outSTR = _getPFNFromLFC(files,
url,
guids,
storageName,
scopeList=scopeList,
tmpLog=tmpLog)
regTime = datetime.datetime.utcnow() - timeStart
tmpLog.debug(
'file lookup for %s LFNs from %s took %s.%03d sec' %
(len(files), url, regTime.seconds, regTime.microseconds / 1000))
# get PFN
if getPFN:
outPFN = {}
try:
if not outSTR in ['', None]:
tmpItems = outSTR.split('LFCRet :')
tmpItems.remove('')
# loop over all returns
for tmpItem in tmpItems:
exec "tmpLFNmap = %s" % tmpItem
for tmpLFN, tmpPFN in tmpLFNmap.iteritems():
outPFN[tmpLFN] = tmpPFN
except:
type, value, traceBack = sys.exc_info()
tmpLog.error(outSTR)
tmpLog.error("could not parse LFC ret - %s %s" % (type, value))
tmpLog.debug('RetPFN:%s files' % len(outPFN))
return outPFN
# check return
if not isinstance(outSTR, types.StringType):
if terminateWhenFailed:
return None
# set empty string
outSTR = ''
# collect OK Files
okFiles = []
for file in files:
if re.search(file, outSTR) != None:
okFiles.append(file)
tmpLog.debug('Ret:%s / %s files' % (str(okFiles[:3]), len(okFiles)))
return okFiles
def _getPFNFromLFC(lfns,
dq2url,
guids,
storageName,
scopeList=[],
tmpLog=None):
if tmpLog == None:
tmpLog = LogWrapper(_log, logPrefix)
tmpLog.debug('_getPFNFromLFC %s %s / %s LFNs:%s %s' %
(dq2url, str(storageName), len(lfns), str(
lfns[:3]), str(scopeList[:3])))
outStr = ''
# check paramter
if guids == [] or storageName == [] or (len(lfns) != len(guids)):
tmpLog.debug('_getPFNFromLFC done with empty list')
return outStr
# check scopeList
if not scopeList in [None, []] and len(lfns) != len(scopeList):
tmpLog.warning('_getPFNFromLFC wrong scopeList %s %s %s %s' %
(dq2url, str(storageName), str(lfns), str(scopeList)))
tmpLog.error('_getPFNFromLFC failed')
return outStr
# loop over all LFNs
iLFN = 0
nLFN = 1000
strFiles = ''
outStr = ''
for iLFN in range(len(lfns)):
if scopeList != []:
strFiles += '%s %s %s\n' % (lfns[iLFN], guids[iLFN],
scopeList[iLFN])
else:
strFiles += '%s %s\n' % (lfns[iLFN], guids[iLFN])
# bulk operation
if (iLFN + 1) % nLFN == 0 or (iLFN + 1) >= len(lfns):
# write to file
inFileName = '%s/lfcin.%s' % (panda_config.logdir,
commands.getoutput('uuidgen'))
ifile = open(inFileName, 'w')
ifile.write(strFiles)
ifile.close()
# construct commands
strStorage = ''
for storage in storageName:
strStorage += '%s,' % storage
strStorage = strStorage[:-1]
com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % (
panda_config.home_dir_cwd, panda_config.home_dir_cwd)
com += 'unset LD_LIBRARY_PATH; unset PYTHONPATH; export PATH=/usr/local/bin:/bin:/usr/bin; '
com+= 'source %s; %s/python -Wignore %s/LFCclient.py -f %s -l %s -s %s' % \
(panda_config.glite_source,panda_config.native_python32,panda_config.lfcClient_dir,
inFileName,dq2url,strStorage)
tmpLog.debug(com)
# exeute
status, output = commands.getstatusoutput(com)
tmpLog.debug(status)
if status == 0:
outStr += output
else:
tmpLog.error("_getPFNFromLFC : %s %s %s" %
(dq2url, status, output))
# send message to logger
try:
# make message
message = 'LFC access : %s %s %s' % (dq2url, status,
output)
# get logger
_pandaLogger = PandaLogger()
_pandaLogger.lock()
_pandaLogger.setParams({'Type': 'broker_util'})
logger = _pandaLogger.getHttpLogger(
panda_config.loggername)
# add message
logger.error(message)
# release HTTP handler
_pandaLogger.release()
except:
pass
tmpLog.error('_getPFNFromLFC failed')
return status
# reset
strFiles = ''
tmpLog.debug('_getPFNFromLFC done')
# return
return outStr