def __init__(self, taskBuffer, jobID, jobStatus, xmlFile, ignoreTmpError=True, siteMapper=None): self.job = None self.jobID = jobID self.jobStatus = jobStatus self.taskBuffer = taskBuffer self.ignoreTmpError = ignoreTmpError self.lockXML = None self.siteMapper = siteMapper self.attemptNr = None self.xmlFile = xmlFile self.datasetMap = {} self.extraInfo = { 'surl': {}, 'nevents': {}, 'lbnr': {}, 'endpoint': {} } # exstract attemptNr try: tmpAttemptNr = self.xmlFile.split('/')[-1].split('_')[-1] if re.search('^\d+$', tmpAttemptNr) != None: self.attemptNr = int(tmpAttemptNr) except: pass # logger self.logger = LogWrapper(_logger, str(self.jobID))
def run(self): # get logger tmpLog = LogWrapper(_logger,'<vuid={0} site={1} name={2}>'.format(self.vuid, self.site, self.dataset)) # query dataset tmpLog.debug("start") if self.vuid != None: dataset = self.taskBuffer.queryDatasetWithMap({'vuid':self.vuid}) else: dataset = self.taskBuffer.queryDatasetWithMap({'name':self.dataset}) if dataset == None: tmpLog.error("Not found") tmpLog.debug("end") return tmpLog.debug("type:%s name:%s" % (dataset.type,dataset.name)) if dataset.type == 'dispatch': # activate jobs in jobsDefined Activator(self.taskBuffer,dataset).start() if dataset.type == 'output': if dataset.name != None and re.search('^panda\..*_zip$',dataset.name) != None: # start unmerge jobs Activator(self.taskBuffer,dataset,enforce=True).start() else: # finish transferring jobs Finisher(self.taskBuffer,dataset,site=self.site).start() tmpLog.debug("end")
def make_logger(tmp_log, token=None, method_name=None, hook=None): # get method name of caller if method_name is None: tmpStr = inspect.stack()[1][3] else: tmpStr = method_name if token is not None: tmpStr += ' <{0}>'.format(token) else: tmpStr += ' :'.format(token) newLog = LogWrapper(tmp_log, tmpStr, seeMem=with_memory_profile, hook=hook) return newLog
def uploadLog(req, file): if not Protocol.isSecure(req): return False if '/CN=limited proxy' in req.subprocess_env['SSL_CLIENT_S_DN']: return False tmpLog = LogWrapper(_logger, 'uploadLog <{0}>'.format(file.filename)) tmpLog.debug("start {0}".format(req.subprocess_env['SSL_CLIENT_S_DN'])) # size check sizeLimit = 100 * 1024 * 1024 # get file size contentLength = 0 try: contentLength = long(req.headers_in["content-length"]) except: if req.headers_in.has_key("content-length"): tmpLog.error("cannot get CL : %s" % req.headers_in["content-length"]) else: tmpLog.error("no CL") tmpLog.debug("size %s" % contentLength) if contentLength > sizeLimit: errStr = "failed to upload log due to size limit" tmpLog.error(errStr) tmpLog.debug("end") return errStr jediLogDir = '/jedilog' retStr = '' try: fileBaseName = file.filename.split('/')[-1] fileFullPath = '{0}{1}/{2}'.format(panda_config.cache_dir, jediLogDir, fileBaseName) # delete old file if os.path.exists(fileFullPath): os.remove(fileFullPath) # write fo = open(fileFullPath, 'wb') fileContent = file.file.read() fo.write(fileContent) fo.close() tmpLog.debug("written to {0}".format(fileFullPath)) retStr = 'http://{0}/cache{1}/{2}'.format(getServerHTTP(None), jediLogDir, fileBaseName) except: errtype, errvalue = sys.exc_info()[:2] errStr = "failed to write log with {0}:{1}".format( errtype.__name__, errvalue) tmpLog.error(errStr) tmpLog.debug("end") return errStr tmpLog.debug("end") return retStr
def __init__(self,taskBuffer,siteMapper,evpFileName,ignoreError): self.taskBuffer = taskBuffer self.siteMapper = siteMapper self.ignoreError = ignoreError self.evpFileName = evpFileName self.token = datetime.datetime.utcnow().isoformat(' ') # logger self.logger = LogWrapper(_logger,self.token) self.pd2p = DynDataDistributer.DynDataDistributer([],self.taskBuffer,self.siteMapper, token=' ',logger=self.logger) self.userDatasetName = '' self.creationTime = '' self.params = '' self.lockedBy = '' self.evpFile = None self.userTaskName = '' # message buffer self.msgBuffer = [] self.lineLimit = 100 # JEDI self.jediTaskID = None
import multiprocessing from taskbuffer.TaskBuffer import taskBuffer import pandalogger.PandaLogger from pandalogger.PandaLogger import PandaLogger from brokerage.SiteMapper import SiteMapper from pandautils import PandaUtils from pandalogger.LogWrapper import LogWrapper # password from config import panda_config passwd = panda_config.dbpasswd # logger _logger = PandaLogger().getLogger('add') tmpLog = LogWrapper(_logger, None) tmpLog.debug("===================== start =====================") # overall timeout value overallTimeout = 20 # grace period try: gracePeriod = int(sys.argv[1]) except: gracePeriod = 3 # current minute currentMinute = datetime.datetime.utcnow().minute
import re from config import panda_config from taskbuffer.TaskBuffer import taskBuffer from pandalogger.PandaLogger import PandaLogger from pandalogger.LogWrapper import LogWrapper import panda_proxy_cache # logger _logger = PandaLogger().getLogger('panda_activeusers_query') tmpLog = LogWrapper(_logger) if __name__ == '__main__': tmpLog.debug("================= start ==================") # instantiate TB taskBuffer.init(panda_config.dbhost, panda_config.dbpasswd, nDBConnection=1) # instantiate MyProxy I/F my_proxy_interface_instance = panda_proxy_cache.MyProxyInterface() # roles if hasattr(panda_config, 'proxy_cache_roles'): roles = panda_config.proxy_cache_roles.split(',') else: roles = [ 'atlas', 'atlas:/atlas/Role=production', 'atlas:/atlas/Role=pilot'
def updateJob(req, jobId, state, token=None, transExitCode=None, pilotErrorCode=None, pilotErrorDiag=None, timestamp=None, timeout=60, xml='', node=None, workdir=None, cpuConsumptionTime=None, cpuConsumptionUnit=None, remainingSpace=None, schedulerID=None, pilotID=None, siteName=None, messageLevel=None, pilotLog='', metaData='', cpuConversionFactor=None, exeErrorCode=None, exeErrorDiag=None, pilotTiming=None, computingElement=None, startTime=None, endTime=None, nEvents=None, nInputFiles=None, batchID=None, attemptNr=None, jobMetrics=None, stdout='', jobSubStatus=None, coreCount=None, maxRSS=None, maxVMEM=None, maxSWAP=None, maxPSS=None, avgRSS=None, avgVMEM=None, avgSWAP=None, avgPSS=None): tmpLog = LogWrapper( _logger, 'updateJob PandaID={0} PID={1}'.format(jobId, os.getpid())) tmpLog.debug('start') # get DN realDN = _getDN(req) # get FQANs fqans = _getFQAN(req) # check production role prodManager = _checkRole(fqans, realDN, jobDispatcher, site=siteName, hostname=req.get_remote_host()) # check token validToken = _checkToken(token, jobDispatcher) # accept json acceptJson = req.acceptJson() _logger.debug( "updateJob(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,attemptNr:%s,jobSubStatus:%s,core:%s,DN:%s,role:%s,token:%s,val:%s,FQAN:%s,maxRSS=%s,maxVMEM=%s,maxSWAP=%s,maxPSS=%s,avgRSS=%s,avgVMEM=%s,avgSWAP=%s,avgPSS=%s\n==XML==\n%s\n==LOG==\n%s\n==Meta==\n%s\n==Metrics==\n%s\n==stdout==\n%s)" % (jobId, state, transExitCode, pilotErrorCode, pilotErrorDiag, node, workdir, cpuConsumptionTime, cpuConsumptionUnit, remainingSpace, schedulerID, pilotID, siteName, messageLevel, nEvents, nInputFiles, cpuConversionFactor, exeErrorCode, exeErrorDiag, pilotTiming, computingElement, startTime, endTime, batchID, attemptNr, jobSubStatus, coreCount, realDN, prodManager, token, validToken, str(fqans), maxRSS, maxVMEM, maxSWAP, maxPSS, avgRSS, avgVMEM, avgSWAP, avgPSS, xml, pilotLog, metaData, jobMetrics, stdout)) _pilotReqLogger.info('method=updateJob,site=%s,node=%s,type=None' % (siteName, node)) # invalid role if not prodManager: _logger.warning("updateJob(%s) : invalid role" % jobId) return Protocol.Response(Protocol.SC_Role).encode(acceptJson) # invalid token if not validToken: _logger.warning("updateJob(%s) : invalid token" % jobId) return Protocol.Response(Protocol.SC_Invalid).encode(acceptJson) # aborting message if jobId == 'NULL': return Protocol.Response(Protocol.SC_Success).encode(acceptJson) # check status if not state in [ 'running', 'failed', 'finished', 'holding', 'starting', 'transferring' ]: _logger.warning("invalid state=%s for updateJob" % state) return Protocol.Response(Protocol.SC_Success).encode(acceptJson) # pilot log tmpLog.debug('sending log') if pilotLog != '': try: # make message message = pilotLog # get logger _pandaLogger = PandaLogger() _pandaLogger.lock() _pandaLogger.setParams({'Type': 'pilotLog', 'PandaID': int(jobId)}) logger = _pandaLogger.getHttpLogger(panda_config.loggername) # add message logger.info(message) except: tmpLog.debug('failed to send log') finally: tmpLog.debug('release lock') try: # release HTTP handler _pandaLogger.release() except: pass tmpLog.debug('done log') # create parameter map param = {} if cpuConsumptionTime != None: param['cpuConsumptionTime'] = cpuConsumptionTime if cpuConsumptionUnit != None: param['cpuConsumptionUnit'] = cpuConsumptionUnit if node != None: param['modificationHost'] = node[:128] if transExitCode != None: param['transExitCode'] = transExitCode if pilotErrorCode != None: param['pilotErrorCode'] = pilotErrorCode if pilotErrorDiag != None: param['pilotErrorDiag'] = pilotErrorDiag[:500] if jobMetrics != None: param['jobMetrics'] = jobMetrics[:500] if schedulerID != None: param['schedulerID'] = schedulerID if pilotID != None: param['pilotID'] = pilotID[:200] if batchID != None: param['batchID'] = batchID[:80] if exeErrorCode != None: param['exeErrorCode'] = exeErrorCode if exeErrorDiag != None: param['exeErrorDiag'] = exeErrorDiag[:500] if cpuConversionFactor != None: param['cpuConversion'] = cpuConversionFactor if pilotTiming != None: param['pilotTiming'] = pilotTiming if computingElement != None: param['computingElement'] = computingElement if nEvents != None: param['nEvents'] = nEvents if nInputFiles != None: param['nInputFiles'] = nInputFiles if not jobSubStatus in [None, '']: param['jobSubStatus'] = jobSubStatus if not coreCount in [None, '']: param['actualCoreCount'] = coreCount if maxRSS != None: param['maxRSS'] = maxRSS if maxVMEM != None: param['maxVMEM'] = maxVMEM if maxSWAP != None: param['maxSWAP'] = maxSWAP if maxPSS != None: param['maxPSS'] = maxPSS if avgRSS != None: param['avgRSS'] = avgRSS if avgVMEM != None: param['avgVMEM'] = avgVMEM if avgSWAP != None: param['avgSWAP'] = avgSWAP if avgPSS != None: param['avgPSS'] = avgPSS if startTime != None: try: param['startTime'] = datetime.datetime( *time.strptime(startTime, '%Y-%m-%d %H:%M:%S')[:6]) except: pass if endTime != None: try: param['endTime'] = datetime.datetime( *time.strptime(endTime, '%Y-%m-%d %H:%M:%S')[:6]) except: pass if attemptNr != None: try: attemptNr = int(attemptNr) except: attemptNr = None if stdout != '': stdout = stdout[:2048] # invoke JD tmpLog.debug('executing') return jobDispatcher.updateJob(int(jobId), state, int(timeout), xml, siteName, param, metaData, attemptNr, stdout, acceptJson)
def application(environ, start_response): # get method name methodName = '' if environ.has_key('SCRIPT_NAME'): methodName = environ['SCRIPT_NAME'].split('/')[-1] tmpLog = LogWrapper(_logger, "PID={0} {1}".format(os.getpid(), methodName)) tmpLog.debug("start") regStart = datetime.datetime.utcnow() retType = None # check method name if not methodName in allowedMethods: tmpLog.error("is forbidden") exeRes = "False : %s is forbidden" % methodName else: # get method object tmpMethod = None try: exec "tmpMethod = %s" % methodName except: pass # object not found if tmpMethod == None: tmpLog.error("is undefined") exeRes = "False" else: try: # get params tmpPars = cgi.FieldStorage(environ['wsgi.input'], environ=environ, keep_blank_values=1) # convert to map params = {} for tmpKey in tmpPars.keys(): if tmpPars[tmpKey].file != None and tmpPars[ tmpKey].filename != None: # file params[tmpKey] = tmpPars[tmpKey] else: # string params[tmpKey] = tmpPars.getfirst(tmpKey) if panda_config.entryVerbose: tmpLog.debug("with %s" % str(params.keys())) # dummy request object dummyReq = DummyReq(environ, tmpLog) # exec exeRes = apply(tmpMethod, [dummyReq], params) # extract return type if type(exeRes) == types.DictType: retType = exeRes['type'] exeRes = exeRes['content'] # convert bool to string if exeRes in [True, False]: exeRes = str(exeRes) except Exception as e: tmpLog.error("execution failure : {0}".format(str(e))) errStr = "" for tmpKey, tmpVal in environ.iteritems(): errStr += "%s : %s\n" % (tmpKey, str(tmpVal)) tmpLog.error(errStr) # return internal server error start_response('500 INTERNAL SERVER ERROR', [('Content-Type', 'text/plain')]) return [str(e)] if panda_config.entryVerbose: tmpLog.debug("done") regTime = datetime.datetime.utcnow() - regStart tmpLog.info( "exec_time=%s.%03d sec, return len=%s B" % (regTime.seconds, regTime.microseconds / 1000, len(str(exeRes)))) # return if exeRes == taskbuffer.ErrorCode.EC_NotFound: start_response('404 Not Found', [('Content-Type', 'text/plain')]) return ['not found'] elif isinstance(exeRes, taskbuffer.ErrorCode.EC_Redirect): start_response('302 Redirect', [('Location', exeRes.url)]) return ['redirect'] else: if retType == 'json': start_response('200 OK', [('Content-Type', 'application/json')]) else: start_response('200 OK', [('Content-Type', 'text/plain')]) return [exeRes]
def __init__(self,job,datasets,log): self.jobSpec = job self.datasets = datasets self.tmpLog = LogWrapper(log,"{0} CloserAtlasPlugin".format(self.jobSpec.PandaID))
def updateJob(req, jobId, state, token=None, transExitCode=None, pilotErrorCode=None, pilotErrorDiag=None, timestamp=None, timeout=60, xml='', node=None, workdir=None, cpuConsumptionTime=None, cpuConsumptionUnit=None, remainingSpace=None, schedulerID=None, pilotID=None, siteName=None, messageLevel=None, pilotLog='', metaData='', cpuConversionFactor=None, exeErrorCode=None, exeErrorDiag=None, pilotTiming=None, computingElement=None, startTime=None, endTime=None, nEvents=None, nInputFiles=None, batchID=None, attemptNr=None, jobMetrics=None, stdout='', jobSubStatus=None, coreCount=None, maxRSS=None, maxVMEM=None, maxSWAP=None, maxPSS=None, avgRSS=None, avgVMEM=None, avgSWAP=None, avgPSS=None, totRCHAR=None, totWCHAR=None, totRBYTES=None, totWBYTES=None, rateRCHAR=None, rateWCHAR=None, rateRBYTES=None, rateWBYTES=None): tmpLog = LogWrapper( _logger, 'updateJob PandaID={0} PID={1}'.format(jobId, os.getpid())) tmpLog.debug('start') # get DN realDN = _getDN(req) # get FQANs fqans = _getFQAN(req) # check production role prodManager = _checkRole(fqans, realDN, jobDispatcher, site=siteName, hostname=req.get_remote_host()) # check token validToken = _checkToken(token, jobDispatcher) # accept json acceptJson = req.acceptJson() _logger.debug( "updateJob(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,attemptNr:%s,jobSubStatus:%s,core:%s,DN:%s,role:%s,token:%s,val:%s,FQAN:%s,maxRSS=%s,maxVMEM=%s,maxSWAP=%s,maxPSS=%s,avgRSS=%s,avgVMEM=%s,avgSWAP=%s,avgPSS=%s,totRCHAR=%s,totWCHAR=%s,totRBYTES=%s,totWBYTES=%s,rateRCHAR=%s,rateWCHAR=%s,rateRBYTES=%s,rateWBYTES=%s\n==XML==\n%s\n==LOG==\n%s\n==Meta==\n%s\n==Metrics==\n%s\n==stdout==\n%s)" % (jobId, state, transExitCode, pilotErrorCode, pilotErrorDiag, node, workdir, cpuConsumptionTime, cpuConsumptionUnit, remainingSpace, schedulerID, pilotID, siteName, messageLevel, nEvents, nInputFiles, cpuConversionFactor, exeErrorCode, exeErrorDiag, pilotTiming, computingElement, startTime, endTime, batchID, attemptNr, jobSubStatus, coreCount, realDN, prodManager, token, validToken, str(fqans), maxRSS, maxVMEM, maxSWAP, maxPSS, avgRSS, avgVMEM, avgSWAP, avgPSS, totRCHAR, totWCHAR, totRBYTES, totWBYTES, rateRCHAR, rateWCHAR, rateRBYTES, rateWBYTES, xml, pilotLog[:1024], metaData[:1024], jobMetrics, stdout)) _pilotReqLogger.debug('method=updateJob,site=%s,node=%s,type=None' % (siteName, node)) # invalid role if not prodManager: _logger.warning("updateJob(%s) : invalid role" % jobId) if acceptJson: tmpMsg = 'no production/pilot role in VOMS FQANs or non pilot owner' else: tmpMsg = None return Protocol.Response(Protocol.SC_Role, tmpMsg).encode(acceptJson) # invalid token if not validToken: _logger.warning("updateJob(%s) : invalid token" % jobId) return Protocol.Response(Protocol.SC_Invalid).encode(acceptJson) # aborting message if jobId == 'NULL': return Protocol.Response(Protocol.SC_Success).encode(acceptJson) # check status if not state in [ 'running', 'failed', 'finished', 'holding', 'starting', 'transferring' ]: _logger.warning("invalid state=%s for updateJob" % state) return Protocol.Response(Protocol.SC_Success).encode(acceptJson) # create parameter map param = {} if cpuConsumptionTime != None: param['cpuConsumptionTime'] = cpuConsumptionTime if cpuConsumptionUnit != None: param['cpuConsumptionUnit'] = cpuConsumptionUnit if node != None: param['modificationHost'] = node[:128] if transExitCode != None: param['transExitCode'] = transExitCode if pilotErrorCode != None: param['pilotErrorCode'] = pilotErrorCode if pilotErrorDiag != None: param['pilotErrorDiag'] = pilotErrorDiag[:500] if jobMetrics != None: param['jobMetrics'] = jobMetrics[:500] if schedulerID != None: param['schedulerID'] = schedulerID if pilotID != None: param['pilotID'] = pilotID[:200] if batchID != None: param['batchID'] = batchID[:80] if exeErrorCode != None: param['exeErrorCode'] = exeErrorCode if exeErrorDiag != None: param['exeErrorDiag'] = exeErrorDiag[:500] if cpuConversionFactor != None: param['cpuConversion'] = cpuConversionFactor if pilotTiming != None: param['pilotTiming'] = pilotTiming if computingElement != None: param['computingElement'] = computingElement if nEvents != None: param['nEvents'] = nEvents if nInputFiles != None: param['nInputFiles'] = nInputFiles if not jobSubStatus in [None, '']: param['jobSubStatus'] = jobSubStatus if not coreCount in [None, '']: param['actualCoreCount'] = coreCount if maxRSS != None: param['maxRSS'] = maxRSS if maxVMEM != None: param['maxVMEM'] = maxVMEM if maxSWAP != None: param['maxSWAP'] = maxSWAP if maxPSS != None: param['maxPSS'] = maxPSS if avgRSS != None: param['avgRSS'] = avgRSS if avgVMEM != None: param['avgVMEM'] = avgVMEM if avgSWAP != None: param['avgSWAP'] = avgSWAP if avgPSS != None: param['avgPSS'] = avgPSS if totRCHAR is not None: totRCHAR = int(totRCHAR) / 1024 # convert to kByte totRCHAR = min(10**10 - 1, totRCHAR) # limit to 10 digit param['totRCHAR'] = totRCHAR if totWCHAR is not None: totWCHAR = int(totWCHAR) / 1024 # convert to kByte totWCHAR = min(10**10 - 1, totWCHAR) # limit to 10 digit param['totWCHAR'] = totWCHAR if totRBYTES is not None: totRBYTES = int(totRBYTES) / 1024 # convert to kByte totRBYTES = min(10**10 - 1, totRBYTES) # limit to 10 digit param['totRBYTES'] = totRBYTES if totWBYTES is not None: totWBYTES = int(totWBYTES) / 1024 # convert to kByte totWBYTES = min(10**10 - 1, totWBYTES) # limit to 10 digit param['totWBYTES'] = totWBYTES if rateRCHAR is not None: rateRCHAR = min(10**10 - 1, int(rateRCHAR)) # limit to 10 digit param['rateRCHAR'] = rateRCHAR if rateWCHAR is not None: rateWCHAR = min(10**10 - 1, int(rateWCHAR)) # limit to 10 digit param['rateWCHAR'] = rateWCHAR if rateRBYTES is not None: rateRBYTES = min(10**10 - 1, int(rateRBYTES)) # limit to 10 digit param['rateRBYTES'] = rateRBYTES if rateWBYTES is not None: rateWBYTES = min(10**10 - 1, int(rateWBYTES)) # limit to 10 digit param['rateWBYTES'] = rateWBYTES if startTime != None: try: param['startTime'] = datetime.datetime( *time.strptime(startTime, '%Y-%m-%d %H:%M:%S')[:6]) except: pass if endTime != None: try: param['endTime'] = datetime.datetime( *time.strptime(endTime, '%Y-%m-%d %H:%M:%S')[:6]) except: pass if attemptNr != None: try: attemptNr = int(attemptNr) except: attemptNr = None if stdout != '': stdout = stdout[:2048] # invoke JD tmpLog.debug('executing') return jobDispatcher.updateJob(int(jobId), state, int(timeout), xml, siteName, param, metaData, pilotLog, attemptNr, stdout, acceptJson)
def run(self): try: # make a message instance tmpLog = LogWrapper(_logger, None) # run main procedure in the same process if not self.forkRun: tmpLog.debug('main start') tmpLog.debug('firstSubmission={0}'.format( self.firstSubmission)) # group jobs per VO voJobsMap = {} ddmFreeJobs = [] tmpLog.debug('{0} jobs in total'.format(len(self.jobs))) for tmpJob in self.jobs: # set VO=local for DDM free if tmpJob.destinationSE == 'local': tmpVO = 'local' else: tmpVO = tmpJob.VO # make map if not voJobsMap.has_key(tmpVO): voJobsMap[tmpVO] = [] voJobsMap[tmpVO].append(tmpJob) # loop over all VOs for tmpVO, tmpJobList in voJobsMap.iteritems(): tmpLog.debug('vo={0} has {1} jobs'.format( tmpVO, len(tmpJobList))) # get plugin setupperPluginClass = panda_config.getPlugin( 'setupper_plugins', tmpVO) if setupperPluginClass == None: # use ATLAS plug-in by default from SetupperAtlasPlugin import SetupperAtlasPlugin setupperPluginClass = SetupperAtlasPlugin tmpLog.debug('plugin name -> {0}'.format( setupperPluginClass.__name__)) try: # make plugin setupperPlugin = setupperPluginClass( self.taskBuffer, self.jobs, tmpLog, resubmit=self.resubmit, pandaDDM=self.pandaDDM, ddmAttempt=self.ddmAttempt, onlyTA=self.onlyTA, firstSubmission=self.firstSubmission) # run plugin tmpLog.debug('run plugin') setupperPlugin.run() # go forward if not TA if not self.onlyTA: # update jobs tmpLog.debug('update jobs') self.updateJobs( setupperPlugin.jobs + setupperPlugin.jumboJobs, tmpLog) # execute post process tmpLog.debug('post execute plugin') setupperPlugin.postRun() tmpLog.debug('done plugin') except: errtype, errvalue = sys.exc_info()[:2] tmpLog.error('plugin failed with {0}:{1}'.format( errtype, errvalue)) tmpLog.debug('main end') else: tmpLog.debug('fork start') # write jobs to file import os import cPickle as pickle outFileName = '%s/set.%s_%s' % (panda_config.logdir, self.jobs[0].PandaID, commands.getoutput('uuidgen')) outFile = open(outFileName, 'w') pickle.dump(self.jobs, outFile) outFile.close() # run main procedure in another process because python doesn't release memory com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % ( panda_config.home_dir_cwd, panda_config.home_dir_cwd) com += 'source %s; ' % panda_config.glite_source com += 'env PYTHONPATH=%s:%s %s/python -Wignore %s/dataservice/forkSetupper.py -i %s' % \ (panda_config.pandaCommon_dir,panda_config.pandaPython_dir,panda_config.native_python, panda_config.pandaPython_dir,outFileName) if self.onlyTA: com += " -t" if not self.firstSubmission: com += " -f" tmpLog.debug(com) # exeute status, output = self.taskBuffer.processLimiter.getstatusoutput( com) tmpLog.debug("return from main process: %s %s" % (status, output)) tmpLog.debug('fork end') except: errtype, errvalue = sys.exc_info()[:2] tmpLog.error('master failed with {0}:{1}'.format( errtype, errvalue))
def getGUIDsFromEventIndex(self, runEventList, streamName, amiTags, dataType): comment = ' /* DBProxy.getGUIDsFromEventIndex */' methodName = comment.split(' ')[-2].split('.')[-1] tmpLog = LogWrapper( _logger, methodName + " <streamName={0} amiTags={1} dataType={2}>".format( streamName, amiTags, dataType)) try: # change to list if not amiTags in [None, '']: amiTags = amiTags.replace('*', '.*').split(',') tmpLog.debug("start for {0} events".format(len(runEventList))) # check data type if not dataType in ['RAW', 'ESD', 'AOD']: return False, 'dataType={0} is unsupported'.format(dataType) # sql to insert runs and events sqlRE = "INSERT INTO {0}.TMP_RUN_EVENT_PAIRS (runNumber,eventNumber) ".format( panda_config.schemaEI) sqlRE += "VALUES (:runNumber,:eventNumber) " varMaps = [] for runNumber, eventNumber in runEventList: varMap = {} varMap[':runNumber'] = runNumber varMap[':eventNumber'] = eventNumber varMaps.append(varMap) # begin transaction self.conn.begin() self.cur.arraysize = 100000 # insert runs and events self.cur.executemany(sqlRE + comment, varMaps) # read GUIDs varMap = {} if amiTags in [None, '']: sqlRG = "SELECT runNumber,eventNumber,guid_{0} ".format( dataType) sqlRG += "FROM {0}.V_PANDA_EVPICK_NOAMITAG_MANY ".format( panda_config.schemaEI) else: sqlRG = "SELECT runNumber,eventNumber,guid_{0},amiTag ".format( dataType) sqlRG += "FROM {0}.V_PANDA_EVPICK_AMITAG_MANY ".format( panda_config.schemaEI) if not streamName in [None, '']: sqlRG += "WHERE streamName=:streamName " varMap[':streamName'] = streamName self.cur.execute(sqlRG + comment, varMap) resRG = self.cur.fetchall() # commit if not self._commit(): raise RuntimeError, 'Commit error' retValue = {} keyAmiIdxMap = {} for tmpItem in resRG: if amiTags in [None, '']: runNumber, eventNumber, guid = tmpItem # dummy idxTag = 0 else: runNumber, eventNumber, guid, amiTag = tmpItem # get index number for the AMI tag in the list idxTag = self.getIndexAmiTag(amiTags, amiTag) # didn't match if idxTag == None: continue tmpKey = (runNumber, eventNumber) # use AMI tag in a preference orde if tmpKey in keyAmiIdxMap and keyAmiIdxMap[tmpKey] < idxTag: continue keyAmiIdxMap[tmpKey] = idxTag retValue[tmpKey] = [guid] tmpLog.debug("found {0} events".format(len(retValue))) return True, retValue except: # roll back self._rollback() # error self.dumpErrorMessage(_logger, methodName) return False, None
def getFilesFromLRC(files, url, guids=[], storageName=[], terminateWhenFailed=False, getPFN=False, scopeList=[]): tmpLog = LogWrapper(_log, None) tmpLog.debug('getFilesFromLRC "%s" %s' % (url, str(storageName))) # get PFC outSTR = '' if url.startswith('mysql://'): # from MySQL outSTR = _getPFNFromMySQL(files, url) # get PFN if getPFN: outPFN = {} # FIXME tmpLog.debug('RetPFN:%s ' % str(outPFN)) return outPFN elif url.startswith('http://'): # from HTTP I/F outSTR = _getPoolFileCatalog(files, url) # get PFN if getPFN: outPFN = {} try: if not outSTR in ['', None]: root = xml.dom.minidom.parseString(outSTR) fileNodes = root.getElementsByTagName('File') for file in fileNodes: # get PFN and LFN nodes physical = file.getElementsByTagName('physical')[0] pfnNode = physical.getElementsByTagName('pfn')[0] logical = file.getElementsByTagName('logical')[0] lfnNode = logical.getElementsByTagName('lfn')[0] # convert UTF8 to Raw pfn = str(pfnNode.getAttribute('name')) lfn = str(lfnNode.getAttribute('name')) # assign if not outPFN.has_key(lfn): outPFN[lfn] = [] outPFN[lfn].append(pfn) except: type, value, traceBack = sys.exc_info() tmpLog.error(outSTR) tmpLog.error("could not parse XML - %s %s" % (type, value)) tmpLog.debug('RetPFN:%s ' % str(outPFN)) return outPFN elif url.startswith('lfc://') or url.startswith('rucio://'): # from LFC timeStart = datetime.datetime.utcnow() outSTR = _getPFNFromLFC(files, url, guids, storageName, scopeList=scopeList, tmpLog=tmpLog) regTime = datetime.datetime.utcnow() - timeStart tmpLog.debug( 'file lookup for %s LFNs from %s took %s.%03d sec' % (len(files), url, regTime.seconds, regTime.microseconds / 1000)) # get PFN if getPFN: outPFN = {} try: if not outSTR in ['', None]: tmpItems = outSTR.split('LFCRet :') tmpItems.remove('') # loop over all returns for tmpItem in tmpItems: exec "tmpLFNmap = %s" % tmpItem for tmpLFN, tmpPFN in tmpLFNmap.iteritems(): outPFN[tmpLFN] = tmpPFN except: type, value, traceBack = sys.exc_info() tmpLog.error(outSTR) tmpLog.error("could not parse LFC ret - %s %s" % (type, value)) tmpLog.debug('RetPFN:%s files' % len(outPFN)) return outPFN # check return if not isinstance(outSTR, types.StringType): if terminateWhenFailed: return None # set empty string outSTR = '' # collect OK Files okFiles = [] for file in files: if re.search(file, outSTR) != None: okFiles.append(file) tmpLog.debug('Ret:%s / %s files' % (str(okFiles[:3]), len(okFiles))) return okFiles
def _getPFNFromLFC(lfns, dq2url, guids, storageName, scopeList=[], tmpLog=None): if tmpLog == None: tmpLog = LogWrapper(_log, logPrefix) tmpLog.debug('_getPFNFromLFC %s %s / %s LFNs:%s %s' % (dq2url, str(storageName), len(lfns), str( lfns[:3]), str(scopeList[:3]))) outStr = '' # check paramter if guids == [] or storageName == [] or (len(lfns) != len(guids)): tmpLog.debug('_getPFNFromLFC done with empty list') return outStr # check scopeList if not scopeList in [None, []] and len(lfns) != len(scopeList): tmpLog.warning('_getPFNFromLFC wrong scopeList %s %s %s %s' % (dq2url, str(storageName), str(lfns), str(scopeList))) tmpLog.error('_getPFNFromLFC failed') return outStr # loop over all LFNs iLFN = 0 nLFN = 1000 strFiles = '' outStr = '' for iLFN in range(len(lfns)): if scopeList != []: strFiles += '%s %s %s\n' % (lfns[iLFN], guids[iLFN], scopeList[iLFN]) else: strFiles += '%s %s\n' % (lfns[iLFN], guids[iLFN]) # bulk operation if (iLFN + 1) % nLFN == 0 or (iLFN + 1) >= len(lfns): # write to file inFileName = '%s/lfcin.%s' % (panda_config.logdir, commands.getoutput('uuidgen')) ifile = open(inFileName, 'w') ifile.write(strFiles) ifile.close() # construct commands strStorage = '' for storage in storageName: strStorage += '%s,' % storage strStorage = strStorage[:-1] com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % ( panda_config.home_dir_cwd, panda_config.home_dir_cwd) com += 'unset LD_LIBRARY_PATH; unset PYTHONPATH; export PATH=/usr/local/bin:/bin:/usr/bin; ' com+= 'source %s; %s/python -Wignore %s/LFCclient.py -f %s -l %s -s %s' % \ (panda_config.glite_source,panda_config.native_python32,panda_config.lfcClient_dir, inFileName,dq2url,strStorage) tmpLog.debug(com) # exeute status, output = commands.getstatusoutput(com) tmpLog.debug(status) if status == 0: outStr += output else: tmpLog.error("_getPFNFromLFC : %s %s %s" % (dq2url, status, output)) # send message to logger try: # make message message = 'LFC access : %s %s %s' % (dq2url, status, output) # get logger _pandaLogger = PandaLogger() _pandaLogger.lock() _pandaLogger.setParams({'Type': 'broker_util'}) logger = _pandaLogger.getHttpLogger( panda_config.loggername) # add message logger.error(message) # release HTTP handler _pandaLogger.release() except: pass tmpLog.error('_getPFNFromLFC failed') return status # reset strFiles = '' tmpLog.debug('_getPFNFromLFC done') # return return outStr